From cf7eaa4617295747ee5646c4e2b7e7a16d7c64ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 7 Dec 2021 12:31:04 +0100 Subject: Revert "bpo-28533: Remove asyncore, asynchat, smtpd modules (GH-29521)" (GH-29951) This reverts commit 9bf2cbc4c498812e14f20d86acb61c53928a5a57. --- .github/CODEOWNERS | 2 + Doc/library/asynchat.rst | 213 ++++ Doc/library/asyncore.rst | 360 +++++++ Doc/library/email.rst | 3 + Doc/library/internet.rst | 1 + Doc/library/ipc.rst | 2 + Doc/library/smtpd.rst | 264 +++++ Doc/library/socketserver.rst | 2 +- Doc/license.rst | 27 + Doc/whatsnew/3.11.rst | 9 - Lib/asynchat.py | 315 ++++++ Lib/asyncore.py | 649 +++++++++++++ Lib/smtpd.py | 885 +++++++++++++++++ Lib/test/libregrtest/save_env.py | 12 +- Lib/test/mock_socket.py | 4 +- Lib/test/support/_asynchat.py | 307 ------ Lib/test/support/_asyncore.py | 643 ------------- Lib/test/support/_smtpd.py | 754 --------------- Lib/test/test_asynchat.py | 292 ++++++ Lib/test/test_asyncore.py | 841 ++++++++++++++++ Lib/test/test_ftplib.py | 10 +- Lib/test/test_logging.py | 7 +- Lib/test/test_os.py | 7 +- Lib/test/test_poplib.py | 8 +- Lib/test/test_smtpd.py | 1018 ++++++++++++++++++++ Lib/test/test_smtplib.py | 8 +- Lib/test/test_ssl.py | 5 +- .../2021-11-11-12-59-10.bpo-28533.68mMZa.rst | 2 - .../2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst | 3 - PCbuild/lib.pyproj | 6 + Python/stdlib_module_names.h | 3 + 31 files changed, 4928 insertions(+), 1734 deletions(-) create mode 100644 Doc/library/asynchat.rst create mode 100644 Doc/library/asyncore.rst create mode 100644 Doc/library/smtpd.rst create mode 100644 Lib/asynchat.py create mode 100644 Lib/asyncore.py create mode 100755 Lib/smtpd.py delete mode 100644 Lib/test/support/_asynchat.py delete mode 100644 Lib/test/support/_asyncore.py delete mode 100755 Lib/test/support/_smtpd.py create mode 100644 Lib/test/test_asynchat.py create mode 100644 Lib/test/test_asyncore.py create mode 100644 Lib/test/test_smtpd.py delete mode 100644 Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst delete mode 100644 Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 82f81e3..ce5121e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -130,6 +130,8 @@ Lib/ast.py @isidentical **/*typing* @gvanrossum @Fidget-Spinner +**/*asyncore @giampaolo +**/*asynchat @giampaolo **/*ftplib @giampaolo **/*shutil @giampaolo diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst new file mode 100644 index 0000000..9e51416 --- /dev/null +++ b/Doc/library/asynchat.rst @@ -0,0 +1,213 @@ +:mod:`asynchat` --- Asynchronous socket command/response handler +================================================================ + +.. module:: asynchat + :synopsis: Support for asynchronous command/response protocols. + +.. moduleauthor:: Sam Rushing +.. sectionauthor:: Steve Holden + +**Source code:** :source:`Lib/asynchat.py` + +.. deprecated:: 3.6 + Please use :mod:`asyncio` instead. + +-------------- + +.. note:: + + This module exists for backwards compatibility only. For new code we + recommend using :mod:`asyncio`. + +This module builds on the :mod:`asyncore` infrastructure, simplifying +asynchronous clients and servers and making it easier to handle protocols +whose elements are terminated by arbitrary strings, or are of variable length. +:mod:`asynchat` defines the abstract class :class:`async_chat` that you +subclass, providing implementations of the :meth:`collect_incoming_data` and +:meth:`found_terminator` methods. It uses the same asynchronous loop as +:mod:`asyncore`, and the two types of channel, :class:`asyncore.dispatcher` +and :class:`asynchat.async_chat`, can freely be mixed in the channel map. +Typically an :class:`asyncore.dispatcher` server channel generates new +:class:`asynchat.async_chat` channel objects as it receives incoming +connection requests. + + +.. class:: async_chat() + + This class is an abstract subclass of :class:`asyncore.dispatcher`. To make + practical use of the code you must subclass :class:`async_chat`, providing + meaningful :meth:`collect_incoming_data` and :meth:`found_terminator` + methods. + The :class:`asyncore.dispatcher` methods can be used, although not all make + sense in a message/response context. + + Like :class:`asyncore.dispatcher`, :class:`async_chat` defines a set of + events that are generated by an analysis of socket conditions after a + :c:func:`select` call. Once the polling loop has been started the + :class:`async_chat` object's methods are called by the event-processing + framework with no action on the part of the programmer. + + Two class attributes can be modified, to improve performance, or possibly + even to conserve memory. + + + .. data:: ac_in_buffer_size + + The asynchronous input buffer size (default ``4096``). + + + .. data:: ac_out_buffer_size + + The asynchronous output buffer size (default ``4096``). + + Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to + define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need + have only one method, :meth:`more`, which should return data to be + transmitted on the channel. + The producer indicates exhaustion (*i.e.* that it contains no more data) by + having its :meth:`more` method return the empty bytes object. At this point + the :class:`async_chat` object removes the producer from the queue and starts + using the next producer, if any. When the producer queue is empty the + :meth:`handle_write` method does nothing. You use the channel object's + :meth:`set_terminator` method to describe how to recognize the end of, or + an important breakpoint in, an incoming transmission from the remote + endpoint. + + To build a functioning :class:`async_chat` subclass your input methods + :meth:`collect_incoming_data` and :meth:`found_terminator` must handle the + data that the channel receives asynchronously. The methods are described + below. + + +.. method:: async_chat.close_when_done() + + Pushes a ``None`` on to the producer queue. When this producer is popped off + the queue it causes the channel to be closed. + + +.. method:: async_chat.collect_incoming_data(data) + + Called with *data* holding an arbitrary amount of received data. The + default method, which must be overridden, raises a + :exc:`NotImplementedError` exception. + + +.. method:: async_chat.discard_buffers() + + In emergencies this method will discard any data held in the input and/or + output buffers and the producer queue. + + +.. method:: async_chat.found_terminator() + + Called when the incoming data stream matches the termination condition set + by :meth:`set_terminator`. The default method, which must be overridden, + raises a :exc:`NotImplementedError` exception. The buffered input data + should be available via an instance attribute. + + +.. method:: async_chat.get_terminator() + + Returns the current terminator for the channel. + + +.. method:: async_chat.push(data) + + Pushes data on to the channel's queue to ensure its transmission. + This is all you need to do to have the channel write the data out to the + network, although it is possible to use your own producers in more complex + schemes to implement encryption and chunking, for example. + + +.. method:: async_chat.push_with_producer(producer) + + Takes a producer object and adds it to the producer queue associated with + the channel. When all currently-pushed producers have been exhausted the + channel will consume this producer's data by calling its :meth:`more` + method and send the data to the remote endpoint. + + +.. method:: async_chat.set_terminator(term) + + Sets the terminating condition to be recognized on the channel. ``term`` + may be any of three types of value, corresponding to three different ways + to handle incoming protocol data. + + +-----------+---------------------------------------------+ + | term | Description | + +===========+=============================================+ + | *string* | Will call :meth:`found_terminator` when the | + | | string is found in the input stream | + +-----------+---------------------------------------------+ + | *integer* | Will call :meth:`found_terminator` when the | + | | indicated number of characters have been | + | | received | + +-----------+---------------------------------------------+ + | ``None`` | The channel continues to collect data | + | | forever | + +-----------+---------------------------------------------+ + + Note that any data following the terminator will be available for reading + by the channel after :meth:`found_terminator` is called. + + +.. _asynchat-example: + +asynchat Example +---------------- + +The following partial example shows how HTTP requests can be read with +:class:`async_chat`. A web server might create an +:class:`http_request_handler` object for each incoming client connection. +Notice that initially the channel terminator is set to match the blank line at +the end of the HTTP headers, and a flag indicates that the headers are being +read. + +Once the headers have been read, if the request is of type POST (indicating +that further data are present in the input stream) then the +``Content-Length:`` header is used to set a numeric terminator to read the +right amount of data from the channel. + +The :meth:`handle_request` method is called once all relevant input has been +marshalled, after setting the channel terminator to ``None`` to ensure that +any extraneous data sent by the web client are ignored. :: + + + import asynchat + + class http_request_handler(asynchat.async_chat): + + def __init__(self, sock, addr, sessions, log): + asynchat.async_chat.__init__(self, sock=sock) + self.addr = addr + self.sessions = sessions + self.ibuffer = [] + self.obuffer = b"" + self.set_terminator(b"\r\n\r\n") + self.reading_headers = True + self.handling = False + self.cgi_data = None + self.log = log + + def collect_incoming_data(self, data): + """Buffer the data""" + self.ibuffer.append(data) + + def found_terminator(self): + if self.reading_headers: + self.reading_headers = False + self.parse_headers(b"".join(self.ibuffer)) + self.ibuffer = [] + if self.op.upper() == b"POST": + clen = self.headers.getheader("content-length") + self.set_terminator(int(clen)) + else: + self.handling = True + self.set_terminator(None) + self.handle_request() + elif not self.handling: + self.set_terminator(None) # browsers sometimes over-send + self.cgi_data = parse(self.headers, b"".join(self.ibuffer)) + self.handling = True + self.ibuffer = [] + self.handle_request() diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst new file mode 100644 index 0000000..a86518e --- /dev/null +++ b/Doc/library/asyncore.rst @@ -0,0 +1,360 @@ +:mod:`asyncore` --- Asynchronous socket handler +=============================================== + +.. module:: asyncore + :synopsis: A base class for developing asynchronous socket handling + services. + +.. moduleauthor:: Sam Rushing +.. sectionauthor:: Christopher Petrilli +.. sectionauthor:: Steve Holden +.. heavily adapted from original documentation by Sam Rushing + +**Source code:** :source:`Lib/asyncore.py` + +.. deprecated:: 3.6 + Please use :mod:`asyncio` instead. + +-------------- + +.. note:: + + This module exists for backwards compatibility only. For new code we + recommend using :mod:`asyncio`. + +This module provides the basic infrastructure for writing asynchronous socket +service clients and servers. + +There are only two ways to have a program on a single processor do "more than +one thing at a time." Multi-threaded programming is the simplest and most +popular way to do it, but there is another very different technique, that lets +you have nearly all the advantages of multi-threading, without actually using +multiple threads. It's really only practical if your program is largely I/O +bound. If your program is processor bound, then pre-emptive scheduled threads +are probably what you really need. Network servers are rarely processor +bound, however. + +If your operating system supports the :c:func:`select` system call in its I/O +library (and nearly all do), then you can use it to juggle multiple +communication channels at once; doing other work while your I/O is taking +place in the "background." Although this strategy can seem strange and +complex, especially at first, it is in many ways easier to understand and +control than multi-threaded programming. The :mod:`asyncore` module solves +many of the difficult problems for you, making the task of building +sophisticated high-performance network servers and clients a snap. For +"conversational" applications and protocols the companion :mod:`asynchat` +module is invaluable. + +The basic idea behind both modules is to create one or more network +*channels*, instances of class :class:`asyncore.dispatcher` and +:class:`asynchat.async_chat`. Creating the channels adds them to a global +map, used by the :func:`loop` function if you do not provide it with your own +*map*. + +Once the initial channel(s) is(are) created, calling the :func:`loop` function +activates channel service, which continues until the last channel (including +any that have been added to the map during asynchronous service) is closed. + + +.. function:: loop([timeout[, use_poll[, map[,count]]]]) + + Enter a polling loop that terminates after count passes or all open + channels have been closed. All arguments are optional. The *count* + parameter defaults to ``None``, resulting in the loop terminating only when all + channels have been closed. The *timeout* argument sets the timeout + parameter for the appropriate :func:`~select.select` or :func:`~select.poll` + call, measured in seconds; the default is 30 seconds. The *use_poll* + parameter, if true, indicates that :func:`~select.poll` should be used in + preference to :func:`~select.select` (the default is ``False``). + + The *map* parameter is a dictionary whose items are the channels to watch. + As channels are closed they are deleted from their map. If *map* is + omitted, a global map is used. Channels (instances of + :class:`asyncore.dispatcher`, :class:`asynchat.async_chat` and subclasses + thereof) can freely be mixed in the map. + + +.. class:: dispatcher() + + The :class:`dispatcher` class is a thin wrapper around a low-level socket + object. To make it more useful, it has a few methods for event-handling + which are called from the asynchronous loop. Otherwise, it can be treated + as a normal non-blocking socket object. + + The firing of low-level events at certain times or in certain connection + states tells the asynchronous loop that certain higher-level events have + taken place. For example, if we have asked for a socket to connect to + another host, we know that the connection has been made when the socket + becomes writable for the first time (at this point you know that you may + write to it with the expectation of success). The implied higher-level + events are: + + +----------------------+----------------------------------------+ + | Event | Description | + +======================+========================================+ + | ``handle_connect()`` | Implied by the first read or write | + | | event | + +----------------------+----------------------------------------+ + | ``handle_close()`` | Implied by a read event with no data | + | | available | + +----------------------+----------------------------------------+ + | ``handle_accepted()``| Implied by a read event on a listening | + | | socket | + +----------------------+----------------------------------------+ + + During asynchronous processing, each mapped channel's :meth:`readable` and + :meth:`writable` methods are used to determine whether the channel's socket + should be added to the list of channels :c:func:`select`\ ed or + :c:func:`poll`\ ed for read and write events. + + Thus, the set of channel events is larger than the basic socket events. The + full set of methods that can be overridden in your subclass follows: + + + .. method:: handle_read() + + Called when the asynchronous loop detects that a :meth:`read` call on the + channel's socket will succeed. + + + .. method:: handle_write() + + Called when the asynchronous loop detects that a writable socket can be + written. Often this method will implement the necessary buffering for + performance. For example:: + + def handle_write(self): + sent = self.send(self.buffer) + self.buffer = self.buffer[sent:] + + + .. method:: handle_expt() + + Called when there is out of band (OOB) data for a socket connection. This + will almost never happen, as OOB is tenuously supported and rarely used. + + + .. method:: handle_connect() + + Called when the active opener's socket actually makes a connection. Might + send a "welcome" banner, or initiate a protocol negotiation with the + remote endpoint, for example. + + + .. method:: handle_close() + + Called when the socket is closed. + + + .. method:: handle_error() + + Called when an exception is raised and not otherwise handled. The default + version prints a condensed traceback. + + + .. method:: handle_accept() + + Called on listening channels (passive openers) when a connection can be + established with a new remote endpoint that has issued a :meth:`connect` + call for the local endpoint. Deprecated in version 3.2; use + :meth:`handle_accepted` instead. + + .. deprecated:: 3.2 + + + .. method:: handle_accepted(sock, addr) + + Called on listening channels (passive openers) when a connection has been + established with a new remote endpoint that has issued a :meth:`connect` + call for the local endpoint. *sock* is a *new* socket object usable to + send and receive data on the connection, and *addr* is the address + bound to the socket on the other end of the connection. + + .. versionadded:: 3.2 + + + .. method:: readable() + + Called each time around the asynchronous loop to determine whether a + channel's socket should be added to the list on which read events can + occur. The default method simply returns ``True``, indicating that by + default, all channels will be interested in read events. + + + .. method:: writable() + + Called each time around the asynchronous loop to determine whether a + channel's socket should be added to the list on which write events can + occur. The default method simply returns ``True``, indicating that by + default, all channels will be interested in write events. + + + In addition, each channel delegates or extends many of the socket methods. + Most of these are nearly identical to their socket partners. + + + .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM) + + This is identical to the creation of a normal socket, and will use the + same options for creation. Refer to the :mod:`socket` documentation for + information on creating sockets. + + .. versionchanged:: 3.3 + *family* and *type* arguments can be omitted. + + + .. method:: connect(address) + + As with the normal socket object, *address* is a tuple with the first + element the host to connect to, and the second the port number. + + + .. method:: send(data) + + Send *data* to the remote end-point of the socket. + + + .. method:: recv(buffer_size) + + Read at most *buffer_size* bytes from the socket's remote end-point. An + empty bytes object implies that the channel has been closed from the + other end. + + Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though + :func:`select.select` or :func:`select.poll` has reported the socket + ready for reading. + + + .. method:: listen(backlog) + + Listen for connections made to the socket. The *backlog* argument + specifies the maximum number of queued connections and should be at least + 1; the maximum value is system-dependent (usually 5). + + + .. method:: bind(address) + + Bind the socket to *address*. The socket must not already be bound. (The + format of *address* depends on the address family --- refer to the + :mod:`socket` documentation for more information.) To mark + the socket as re-usable (setting the :const:`SO_REUSEADDR` option), call + the :class:`dispatcher` object's :meth:`set_reuse_addr` method. + + + .. method:: accept() + + Accept a connection. The socket must be bound to an address and listening + for connections. The return value can be either ``None`` or a pair + ``(conn, address)`` where *conn* is a *new* socket object usable to send + and receive data on the connection, and *address* is the address bound to + the socket on the other end of the connection. + When ``None`` is returned it means the connection didn't take place, in + which case the server should just ignore this event and keep listening + for further incoming connections. + + + .. method:: close() + + Close the socket. All future operations on the socket object will fail. + The remote end-point will receive no more data (after queued data is + flushed). Sockets are automatically closed when they are + garbage-collected. + + +.. class:: dispatcher_with_send() + + A :class:`dispatcher` subclass which adds simple buffered output capability, + useful for simple clients. For more sophisticated usage use + :class:`asynchat.async_chat`. + +.. class:: file_dispatcher() + + A file_dispatcher takes a file descriptor or :term:`file object` along + with an optional map argument and wraps it for use with the :c:func:`poll` + or :c:func:`loop` functions. If provided a file object or anything with a + :c:func:`fileno` method, that method will be called and passed to the + :class:`file_wrapper` constructor. + + .. availability:: Unix. + +.. class:: file_wrapper() + + A file_wrapper takes an integer file descriptor and calls :func:`os.dup` to + duplicate the handle so that the original handle may be closed independently + of the file_wrapper. This class implements sufficient methods to emulate a + socket for use by the :class:`file_dispatcher` class. + + .. availability:: Unix. + + +.. _asyncore-example-1: + +asyncore Example basic HTTP client +---------------------------------- + +Here is a very basic HTTP client that uses the :class:`dispatcher` class to +implement its socket handling:: + + import asyncore + + class HTTPClient(asyncore.dispatcher): + + def __init__(self, host, path): + asyncore.dispatcher.__init__(self) + self.create_socket() + self.connect( (host, 80) ) + self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' % + (path, host), 'ascii') + + def handle_connect(self): + pass + + def handle_close(self): + self.close() + + def handle_read(self): + print(self.recv(8192)) + + def writable(self): + return (len(self.buffer) > 0) + + def handle_write(self): + sent = self.send(self.buffer) + self.buffer = self.buffer[sent:] + + + client = HTTPClient('www.python.org', '/') + asyncore.loop() + +.. _asyncore-example-2: + +asyncore Example basic echo server +---------------------------------- + +Here is a basic echo server that uses the :class:`dispatcher` class to accept +connections and dispatches the incoming connections to a handler:: + + import asyncore + + class EchoHandler(asyncore.dispatcher_with_send): + + def handle_read(self): + data = self.recv(8192) + if data: + self.send(data) + + class EchoServer(asyncore.dispatcher): + + def __init__(self, host, port): + asyncore.dispatcher.__init__(self) + self.create_socket() + self.set_reuse_addr() + self.bind((host, port)) + self.listen(5) + + def handle_accepted(self, sock, addr): + print('Incoming connection from %s' % repr(addr)) + handler = EchoHandler(sock) + + server = EchoServer('localhost', 8080) + asyncore.loop() diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 816fae9..5eebcd9 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -147,3 +147,6 @@ Legacy API: Module :mod:`mailbox` Tools for creating, reading, and managing collections of messages on disk using a variety standard formats. + + Module :mod:`smtpd` + SMTP server framework (primarily useful for testing) diff --git a/Doc/library/internet.rst b/Doc/library/internet.rst index 65693c9..e745dd1 100644 --- a/Doc/library/internet.rst +++ b/Doc/library/internet.rst @@ -35,6 +35,7 @@ is currently supported on most popular platforms. Here is an overview: imaplib.rst nntplib.rst smtplib.rst + smtpd.rst telnetlib.rst uuid.rst socketserver.rst diff --git a/Doc/library/ipc.rst b/Doc/library/ipc.rst index 4849c82..b88a174 100644 --- a/Doc/library/ipc.rst +++ b/Doc/library/ipc.rst @@ -22,5 +22,7 @@ The list of modules described in this chapter is: ssl.rst select.rst selectors.rst + asyncore.rst + asynchat.rst signal.rst mmap.rst diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst new file mode 100644 index 0000000..611411d --- /dev/null +++ b/Doc/library/smtpd.rst @@ -0,0 +1,264 @@ +:mod:`smtpd` --- SMTP Server +============================ + +.. module:: smtpd + :synopsis: A SMTP server implementation in Python. + +.. moduleauthor:: Barry Warsaw +.. sectionauthor:: Moshe Zadka + +**Source code:** :source:`Lib/smtpd.py` + +-------------- + +This module offers several classes to implement SMTP (email) servers. + +.. deprecated:: 3.6 + The `aiosmtpd `_ package is a recommended + replacement for this module. It is based on :mod:`asyncio` and provides a + more straightforward API. + +Several server implementations are present; one is a generic +do-nothing implementation, which can be overridden, while the other two offer +specific mail-sending strategies. + +Additionally the SMTPChannel may be extended to implement very specific +interaction behaviour with SMTP clients. + +The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531` +SMTPUTF8 extensions. + + +SMTPServer Objects +------------------ + + +.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ + map=None, enable_SMTPUTF8=False, decode_data=False) + + Create a new :class:`SMTPServer` object, which binds to local address + *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. Both + *localaddr* and *remoteaddr* should be a :ref:`(host, port) ` + tuple. The object inherits from :class:`asyncore.dispatcher`, and so will + insert itself into :mod:`asyncore`'s event loop on instantiation. + + *data_size_limit* specifies the maximum number of bytes that will be + accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no + limit. + + *map* is the socket map to use for connections (an initially empty + dictionary is a suitable value). If not specified the :mod:`asyncore` + global socket map is used. + + *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined + in :RFC:`6531`) should be enabled. The default is ``False``. + When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` + command and when present is passed to :meth:`process_message` in the + ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* + cannot be set to ``True`` at the same time. + + *decode_data* specifies whether the data portion of the SMTP transaction + should be decoded using UTF-8. When *decode_data* is ``False`` (the + default), the server advertises the ``8BITMIME`` + extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to + the ``MAIL`` command, and when present passes it to :meth:`process_message` + in the ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* + cannot be set to ``True`` at the same time. + + .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) + + Raise a :exc:`NotImplementedError` exception. Override this in subclasses to + do something useful with this message. Whatever was passed in the + constructor as *remoteaddr* will be available as the :attr:`_remoteaddr` + attribute. *peer* is the remote host's address, *mailfrom* is the envelope + originator, *rcpttos* are the envelope recipients and *data* is a string + containing the contents of the e-mail (which should be in :rfc:`5321` + format). + + If the *decode_data* constructor keyword is set to ``True``, the *data* + argument will be a unicode string. If it is set to ``False``, it + will be a bytes object. + + *kwargs* is a dictionary containing additional information. It is empty + if ``decode_data=True`` was given as an init argument, otherwise + it contains the following keys: + + *mail_options*: + a list of all received parameters to the ``MAIL`` + command (the elements are uppercase strings; example: + ``['BODY=8BITMIME', 'SMTPUTF8']``). + + *rcpt_options*: + same as *mail_options* but for the ``RCPT`` command. + Currently no ``RCPT TO`` options are supported, so for now + this will always be an empty list. + + Implementations of ``process_message`` should use the ``**kwargs`` + signature to accept arbitrary keyword arguments, since future feature + enhancements may add keys to the kwargs dictionary. + + Return ``None`` to request a normal ``250 Ok`` response; otherwise + return the desired response string in :RFC:`5321` format. + + .. attribute:: channel_class + + Override this in subclasses to use a custom :class:`SMTPChannel` for + managing SMTP clients. + + .. versionadded:: 3.4 + The *map* constructor argument. + + .. versionchanged:: 3.5 + *localaddr* and *remoteaddr* may now contain IPv6 addresses. + + .. versionadded:: 3.5 + The *decode_data* and *enable_SMTPUTF8* constructor parameters, and the + *kwargs* parameter to :meth:`process_message` when *decode_data* is + ``False``. + + .. versionchanged:: 3.6 + *decode_data* is now ``False`` by default. + + +DebuggingServer Objects +----------------------- + + +.. class:: DebuggingServer(localaddr, remoteaddr) + + Create a new debugging server. Arguments are as per :class:`SMTPServer`. + Messages will be discarded, and printed on stdout. + + +PureProxy Objects +----------------- + + +.. class:: PureProxy(localaddr, remoteaddr) + + Create a new pure proxy server. Arguments are as per :class:`SMTPServer`. + Everything will be relayed to *remoteaddr*. Note that running this has a good + chance to make you into an open relay, so please be careful. + + +SMTPChannel Objects +------------------- + +.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ + map=None, enable_SMTPUTF8=False, decode_data=False) + + Create a new :class:`SMTPChannel` object which manages the communication + between the server and a single SMTP client. + + *conn* and *addr* are as per the instance variables described below. + + *data_size_limit* specifies the maximum number of bytes that will be + accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no + limit. + + *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined + in :RFC:`6531`) should be enabled. The default is ``False``. + *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same + time. + + A dictionary can be specified in *map* to avoid using a global socket map. + + *decode_data* specifies whether the data portion of the SMTP transaction + should be decoded using UTF-8. The default is ``False``. + *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same + time. + + To use a custom SMTPChannel implementation you need to override the + :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. + + .. versionchanged:: 3.5 + The *decode_data* and *enable_SMTPUTF8* parameters were added. + + .. versionchanged:: 3.6 + *decode_data* is now ``False`` by default. + + The :class:`SMTPChannel` has the following instance variables: + + .. attribute:: smtp_server + + Holds the :class:`SMTPServer` that spawned this channel. + + .. attribute:: conn + + Holds the socket object connecting to the client. + + .. attribute:: addr + + Holds the address of the client, the second value returned by + :func:`socket.accept ` + + .. attribute:: received_lines + + Holds a list of the line strings (decoded using UTF-8) received from + the client. The lines have their ``"\r\n"`` line ending translated to + ``"\n"``. + + .. attribute:: smtp_state + + Holds the current state of the channel. This will be either + :attr:`COMMAND` initially and then :attr:`DATA` after the client sends + a "DATA" line. + + .. attribute:: seen_greeting + + Holds a string containing the greeting sent by the client in its "HELO". + + .. attribute:: mailfrom + + Holds a string containing the address identified in the "MAIL FROM:" line + from the client. + + .. attribute:: rcpttos + + Holds a list of strings containing the addresses identified in the + "RCPT TO:" lines from the client. + + .. attribute:: received_data + + Holds a string containing all of the data sent by the client during the + DATA state, up to but not including the terminating ``"\r\n.\r\n"``. + + .. attribute:: fqdn + + Holds the fully-qualified domain name of the server as returned by + :func:`socket.getfqdn`. + + .. attribute:: peer + + Holds the name of the client peer as returned by ``conn.getpeername()`` + where ``conn`` is :attr:`conn`. + + The :class:`SMTPChannel` operates by invoking methods named ``smtp_`` + upon reception of a command line from the client. Built into the base + :class:`SMTPChannel` class are methods for handling the following commands + (and responding to them appropriately): + + ======== =================================================================== + Command Action taken + ======== =================================================================== + HELO Accepts the greeting from the client and stores it in + :attr:`seen_greeting`. Sets server to base command mode. + EHLO Accepts the greeting from the client and stores it in + :attr:`seen_greeting`. Sets server to extended command mode. + NOOP Takes no action. + QUIT Closes the connection cleanly. + MAIL Accepts the "MAIL FROM:" syntax and stores the supplied address as + :attr:`mailfrom`. In extended command mode, accepts the + :rfc:`1870` SIZE attribute and responds appropriately based on the + value of *data_size_limit*. + RCPT Accepts the "RCPT TO:" syntax and stores the supplied addresses in + the :attr:`rcpttos` list. + RSET Resets the :attr:`mailfrom`, :attr:`rcpttos`, and + :attr:`received_data`, but not the greeting. + DATA Sets the internal state to :attr:`DATA` and stores remaining lines + from the client in :attr:`received_data` until the terminator + ``"\r\n.\r\n"`` is received. + HELP Returns minimal information on command syntax + VRFY Returns code 252 (the server doesn't know if the address is valid) + EXPN Reports that the command is not implemented. + ======== =================================================================== diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 8aa72bd..b65a3e8 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -176,7 +176,7 @@ partially finished requests and to use :mod:`selectors` to decide which request to work on next (or whether to handle a new incoming request). This is particularly important for stream services where each client can potentially be connected for a long time (if threads or subprocesses cannot be used). See -:mod:`asyncio` for another way to manage this. +:mod:`asyncore` for another way to manage this. .. XXX should data and methods be intermingled, or separate? how should the distinction between class and instance variables be drawn? diff --git a/Doc/license.rst b/Doc/license.rst index 1d086b6..cd03411 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -383,6 +383,33 @@ Project, http://www.wide.ad.jp/. :: SUCH DAMAGE. +Asynchronous socket services +---------------------------- + +The :mod:`asynchat` and :mod:`asyncore` modules contain the following notice:: + + Copyright 1996 by Sam Rushing + + All Rights Reserved + + Permission to use, copy, modify, and distribute this software and + its documentation for any purpose and without fee is hereby + granted, provided that the above copyright notice appear in all + copies and that both that copyright notice and this permission + notice appear in supporting documentation, and that the name of Sam + Rushing not be used in advertising or publicity pertaining to + distribution of the software without specific, written prior + permission. + + SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN + NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR + CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, + NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + Cookie management ----------------- diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index b06d8d4..a570864 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -518,15 +518,6 @@ Removed (Contributed by Hugo van Kemenade in :issue:`45320`.) -* Remove the ``asyncore`` and ``asynchat`` modules, deprecated in Python 3.6: - use the :mod:`asyncio` module instead. - (Contributed by Victor Stinner in :issue:`28533`.) - -* Remove the ``smtpd`` module, deprecated in Python 3.6: the `aiosmtpd - `__ module can be used instead, it is based - on asyncio. - (Contributed by Victor Stinner in :issue:`28533`.) - Porting to Python 3.11 ====================== diff --git a/Lib/asynchat.py b/Lib/asynchat.py new file mode 100644 index 0000000..de26ffa --- /dev/null +++ b/Lib/asynchat.py @@ -0,0 +1,315 @@ +# -*- Mode: Python; tab-width: 4 -*- +# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +r"""A class supporting chat-style (command/response) protocols. + +This class adds support for 'chat' style protocols - where one side +sends a 'command', and the other sends a response (examples would be +the common internet protocols - smtp, nntp, ftp, etc..). + +The handle_read() method looks at the input stream for the current +'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' +for multi-line output), calling self.found_terminator() on its +receipt. + +for example: +Say you build an async nntp client using this class. At the start +of the connection, you'll have self.terminator set to '\r\n', in +order to process the single-line greeting. Just before issuing a +'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST +command will be accumulated (using your own 'collect_incoming_data' +method) up to the terminator, and then control will be returned to +you - by calling your self.found_terminator() method. +""" +import asyncore +from collections import deque + +from warnings import warn +warn( + 'The asynchat module is deprecated. ' + 'The recommended replacement is asyncio', + DeprecationWarning, + stacklevel=2) + + + +class async_chat(asyncore.dispatcher): + """This is an abstract class. You must derive from this class, and add + the two methods collect_incoming_data() and found_terminator()""" + + # these are overridable defaults + + ac_in_buffer_size = 65536 + ac_out_buffer_size = 65536 + + # we don't want to enable the use of encoding by default, because that is a + # sign of an application bug that we don't want to pass silently + + use_encoding = 0 + encoding = 'latin-1' + + def __init__(self, sock=None, map=None): + # for string terminator matching + self.ac_in_buffer = b'' + + # we use a list here rather than io.BytesIO for a few reasons... + # del lst[:] is faster than bio.truncate(0) + # lst = [] is faster than bio.truncate(0) + self.incoming = [] + + # we toss the use of the "simple producer" and replace it with + # a pure deque, which the original fifo was a wrapping of + self.producer_fifo = deque() + asyncore.dispatcher.__init__(self, sock, map) + + def collect_incoming_data(self, data): + raise NotImplementedError("must be implemented in subclass") + + def _collect_incoming_data(self, data): + self.incoming.append(data) + + def _get_data(self): + d = b''.join(self.incoming) + del self.incoming[:] + return d + + def found_terminator(self): + raise NotImplementedError("must be implemented in subclass") + + def set_terminator(self, term): + """Set the input delimiter. + + Can be a fixed string of any length, an integer, or None. + """ + if isinstance(term, str) and self.use_encoding: + term = bytes(term, self.encoding) + elif isinstance(term, int) and term < 0: + raise ValueError('the number of received bytes must be positive') + self.terminator = term + + def get_terminator(self): + return self.terminator + + # grab some more data from the socket, + # throw it to the collector method, + # check for the terminator, + # if found, transition to the next state. + + def handle_read(self): + + try: + data = self.recv(self.ac_in_buffer_size) + except BlockingIOError: + return + except OSError: + self.handle_error() + return + + if isinstance(data, str) and self.use_encoding: + data = bytes(str, self.encoding) + self.ac_in_buffer = self.ac_in_buffer + data + + # Continue to search for self.terminator in self.ac_in_buffer, + # while calling self.collect_incoming_data. The while loop + # is necessary because we might read several data+terminator + # combos with a single recv(4096). + + while self.ac_in_buffer: + lb = len(self.ac_in_buffer) + terminator = self.get_terminator() + if not terminator: + # no terminator, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + elif isinstance(terminator, int): + # numeric terminator + n = terminator + if lb < n: + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + self.terminator = self.terminator - lb + else: + self.collect_incoming_data(self.ac_in_buffer[:n]) + self.ac_in_buffer = self.ac_in_buffer[n:] + self.terminator = 0 + self.found_terminator() + else: + # 3 cases: + # 1) end of buffer matches terminator exactly: + # collect data, transition + # 2) end of buffer matches some prefix: + # collect data to the prefix + # 3) end of buffer does not match any prefix: + # collect data + terminator_len = len(terminator) + index = self.ac_in_buffer.find(terminator) + if index != -1: + # we found the terminator + if index > 0: + # don't bother reporting the empty string + # (source of subtle bugs) + self.collect_incoming_data(self.ac_in_buffer[:index]) + self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] + # This does the Right Thing if the terminator + # is changed here. + self.found_terminator() + else: + # check for a prefix of the terminator + index = find_prefix_at_end(self.ac_in_buffer, terminator) + if index: + if index != lb: + # we found a prefix, collect up to the prefix + self.collect_incoming_data(self.ac_in_buffer[:-index]) + self.ac_in_buffer = self.ac_in_buffer[-index:] + break + else: + # no prefix, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + + def handle_write(self): + self.initiate_send() + + def handle_close(self): + self.close() + + def push(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) + sabs = self.ac_out_buffer_size + if len(data) > sabs: + for i in range(0, len(data), sabs): + self.producer_fifo.append(data[i:i+sabs]) + else: + self.producer_fifo.append(data) + self.initiate_send() + + def push_with_producer(self, producer): + self.producer_fifo.append(producer) + self.initiate_send() + + def readable(self): + "predicate for inclusion in the readable for select()" + # cannot use the old predicate, it violates the claim of the + # set_terminator method. + + # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) + return 1 + + def writable(self): + "predicate for inclusion in the writable for select()" + return self.producer_fifo or (not self.connected) + + def close_when_done(self): + "automatically close this channel once the outgoing queue is empty" + self.producer_fifo.append(None) + + def initiate_send(self): + while self.producer_fifo and self.connected: + first = self.producer_fifo[0] + # handle empty string/buffer or None entry + if not first: + del self.producer_fifo[0] + if first is None: + self.handle_close() + return + + # handle classic producer behavior + obs = self.ac_out_buffer_size + try: + data = first[:obs] + except TypeError: + data = first.more() + if data: + self.producer_fifo.appendleft(data) + else: + del self.producer_fifo[0] + continue + + if isinstance(data, str) and self.use_encoding: + data = bytes(data, self.encoding) + + # send the data + try: + num_sent = self.send(data) + except OSError: + self.handle_error() + return + + if num_sent: + if num_sent < len(data) or obs < len(first): + self.producer_fifo[0] = first[num_sent:] + else: + del self.producer_fifo[0] + # we tried to send some actual data + return + + def discard_buffers(self): + # Emergencies only! + self.ac_in_buffer = b'' + del self.incoming[:] + self.producer_fifo.clear() + + +class simple_producer: + + def __init__(self, data, buffer_size=512): + self.data = data + self.buffer_size = buffer_size + + def more(self): + if len(self.data) > self.buffer_size: + result = self.data[:self.buffer_size] + self.data = self.data[self.buffer_size:] + return result + else: + result = self.data + self.data = b'' + return result + + +# Given 'haystack', see if any prefix of 'needle' is at its end. This +# assumes an exact match has already been checked. Return the number of +# characters matched. +# for example: +# f_p_a_e("qwerty\r", "\r\n") => 1 +# f_p_a_e("qwertydkjf", "\r\n") => 0 +# f_p_a_e("qwerty\r\n", "\r\n") => + +# this could maybe be made faster with a computed regex? +# [answer: no; circa Python-2.0, Jan 2001] +# new python: 28961/s +# old python: 18307/s +# re: 12820/s +# regex: 14035/s + +def find_prefix_at_end(haystack, needle): + l = len(needle) - 1 + while l and not haystack.endswith(needle[:l]): + l -= 1 + return l diff --git a/Lib/asyncore.py b/Lib/asyncore.py new file mode 100644 index 0000000..b1eea4b --- /dev/null +++ b/Lib/asyncore.py @@ -0,0 +1,649 @@ +# -*- Mode: Python -*- +# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +"""Basic infrastructure for asynchronous socket service clients and servers. + +There are only two ways to have a program on a single processor do "more +than one thing at a time". Multi-threaded programming is the simplest and +most popular way to do it, but there is another very different technique, +that lets you have nearly all the advantages of multi-threading, without +actually using multiple threads. it's really only practical if your program +is largely I/O bound. If your program is CPU bound, then pre-emptive +scheduled threads are probably what you really need. Network servers are +rarely CPU-bound, however. + +If your operating system supports the select() system call in its I/O +library (and nearly all do), then you can use it to juggle multiple +communication channels at once; doing other work while your I/O is taking +place in the "background." Although this strategy can seem strange and +complex, especially at first, it is in many ways easier to understand and +control than multi-threaded programming. The module documented here solves +many of the difficult problems for you, making the task of building +sophisticated high-performance network servers and clients a snap. +""" + +import select +import socket +import sys +import time +import warnings + +import os +from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ + ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + +warnings.warn( + 'The asyncore module is deprecated. ' + 'The recommended replacement is asyncio', + DeprecationWarning, + stacklevel=2) + + +_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF}) + +try: + socket_map +except NameError: + socket_map = {} + +def _strerror(err): + try: + return os.strerror(err) + except (ValueError, OverflowError, NameError): + if err in errorcode: + return errorcode[err] + return "Unknown error %s" %err + +class ExitNow(Exception): + pass + +_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) + +def read(obj): + try: + obj.handle_read_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def write(obj): + try: + obj.handle_write_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def _exception(obj): + try: + obj.handle_expt_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def readwrite(obj, flags): + try: + if flags & select.POLLIN: + obj.handle_read_event() + if flags & select.POLLOUT: + obj.handle_write_event() + if flags & select.POLLPRI: + obj.handle_expt_event() + if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): + obj.handle_close() + except OSError as e: + if e.errno not in _DISCONNECTED: + obj.handle_error() + else: + obj.handle_close() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def poll(timeout=0.0, map=None): + if map is None: + map = socket_map + if map: + r = []; w = []; e = [] + for fd, obj in list(map.items()): + is_r = obj.readable() + is_w = obj.writable() + if is_r: + r.append(fd) + # accepting sockets should not be writable + if is_w and not obj.accepting: + w.append(fd) + if is_r or is_w: + e.append(fd) + if [] == r == w == e: + time.sleep(timeout) + return + + r, w, e = select.select(r, w, e, timeout) + + for fd in r: + obj = map.get(fd) + if obj is None: + continue + read(obj) + + for fd in w: + obj = map.get(fd) + if obj is None: + continue + write(obj) + + for fd in e: + obj = map.get(fd) + if obj is None: + continue + _exception(obj) + +def poll2(timeout=0.0, map=None): + # Use the poll() support added to the select module in Python 2.0 + if map is None: + map = socket_map + if timeout is not None: + # timeout is in milliseconds + timeout = int(timeout*1000) + pollster = select.poll() + if map: + for fd, obj in list(map.items()): + flags = 0 + if obj.readable(): + flags |= select.POLLIN | select.POLLPRI + # accepting sockets should not be writable + if obj.writable() and not obj.accepting: + flags |= select.POLLOUT + if flags: + pollster.register(fd, flags) + + r = pollster.poll(timeout) + for fd, flags in r: + obj = map.get(fd) + if obj is None: + continue + readwrite(obj, flags) + +poll3 = poll2 # Alias for backward compatibility + +def loop(timeout=30.0, use_poll=False, map=None, count=None): + if map is None: + map = socket_map + + if use_poll and hasattr(select, 'poll'): + poll_fun = poll2 + else: + poll_fun = poll + + if count is None: + while map: + poll_fun(timeout, map) + + else: + while map and count > 0: + poll_fun(timeout, map) + count = count - 1 + +class dispatcher: + + debug = False + connected = False + accepting = False + connecting = False + closing = False + addr = None + ignore_log_types = frozenset({'warning'}) + + def __init__(self, sock=None, map=None): + if map is None: + self._map = socket_map + else: + self._map = map + + self._fileno = None + + if sock: + # Set to nonblocking just to make sure for cases where we + # get a socket from a blocking source. + sock.setblocking(False) + self.set_socket(sock, map) + self.connected = True + # The constructor no longer requires that the socket + # passed be connected. + try: + self.addr = sock.getpeername() + except OSError as err: + if err.errno in (ENOTCONN, EINVAL): + # To handle the case where we got an unconnected + # socket. + self.connected = False + else: + # The socket is broken in some unknown way, alert + # the user and remove it from the map (to prevent + # polling of broken sockets). + self.del_channel(map) + raise + else: + self.socket = None + + def __repr__(self): + status = [self.__class__.__module__+"."+self.__class__.__qualname__] + if self.accepting and self.addr: + status.append('listening') + elif self.connected: + status.append('connected') + if self.addr is not None: + try: + status.append('%s:%d' % self.addr) + except TypeError: + status.append(repr(self.addr)) + return '<%s at %#x>' % (' '.join(status), id(self)) + + def add_channel(self, map=None): + #self.log_info('adding channel %s' % self) + if map is None: + map = self._map + map[self._fileno] = self + + def del_channel(self, map=None): + fd = self._fileno + if map is None: + map = self._map + if fd in map: + #self.log_info('closing channel %d:%s' % (fd, self)) + del map[fd] + self._fileno = None + + def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): + self.family_and_type = family, type + sock = socket.socket(family, type) + sock.setblocking(False) + self.set_socket(sock) + + def set_socket(self, sock, map=None): + self.socket = sock + self._fileno = sock.fileno() + self.add_channel(map) + + def set_reuse_addr(self): + # try to re-use a server port if possible + try: + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, + self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR) | 1 + ) + except OSError: + pass + + # ================================================== + # predicates for select() + # these are used as filters for the lists of sockets + # to pass to select(). + # ================================================== + + def readable(self): + return True + + def writable(self): + return True + + # ================================================== + # socket object methods. + # ================================================== + + def listen(self, num): + self.accepting = True + if os.name == 'nt' and num > 5: + num = 5 + return self.socket.listen(num) + + def bind(self, addr): + self.addr = addr + return self.socket.bind(addr) + + def connect(self, address): + self.connected = False + self.connecting = True + err = self.socket.connect_ex(address) + if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ + or err == EINVAL and os.name == 'nt': + self.addr = address + return + if err in (0, EISCONN): + self.addr = address + self.handle_connect_event() + else: + raise OSError(err, errorcode[err]) + + def accept(self): + # XXX can return either an address pair or None + try: + conn, addr = self.socket.accept() + except TypeError: + return None + except OSError as why: + if why.errno in (EWOULDBLOCK, ECONNABORTED, EAGAIN): + return None + else: + raise + else: + return conn, addr + + def send(self, data): + try: + result = self.socket.send(data) + return result + except OSError as why: + if why.errno == EWOULDBLOCK: + return 0 + elif why.errno in _DISCONNECTED: + self.handle_close() + return 0 + else: + raise + + def recv(self, buffer_size): + try: + data = self.socket.recv(buffer_size) + if not data: + # a closed connection is indicated by signaling + # a read condition, and having recv() return 0. + self.handle_close() + return b'' + else: + return data + except OSError as why: + # winsock sometimes raises ENOTCONN + if why.errno in _DISCONNECTED: + self.handle_close() + return b'' + else: + raise + + def close(self): + self.connected = False + self.accepting = False + self.connecting = False + self.del_channel() + if self.socket is not None: + try: + self.socket.close() + except OSError as why: + if why.errno not in (ENOTCONN, EBADF): + raise + + # log and log_info may be overridden to provide more sophisticated + # logging and warning methods. In general, log is for 'hit' logging + # and 'log_info' is for informational, warning and error logging. + + def log(self, message): + sys.stderr.write('log: %s\n' % str(message)) + + def log_info(self, message, type='info'): + if type not in self.ignore_log_types: + print('%s: %s' % (type, message)) + + def handle_read_event(self): + if self.accepting: + # accepting sockets are never connected, they "spawn" new + # sockets that are connected + self.handle_accept() + elif not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_read() + else: + self.handle_read() + + def handle_connect_event(self): + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + raise OSError(err, _strerror(err)) + self.handle_connect() + self.connected = True + self.connecting = False + + def handle_write_event(self): + if self.accepting: + # Accepting sockets shouldn't get a write event. + # We will pretend it didn't happen. + return + + if not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_write() + + def handle_expt_event(self): + # handle_expt_event() is called if there might be an error on the + # socket, or if there is OOB data + # check for the error condition first + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + # we can get here when select.select() says that there is an + # exceptional condition on the socket + # since there is an error, we'll go ahead and close the socket + # like we would in a subclassed handle_read() that received no + # data + self.handle_close() + else: + self.handle_expt() + + def handle_error(self): + nil, t, v, tbinfo = compact_traceback() + + # sometimes a user repr method will crash. + try: + self_repr = repr(self) + except: + self_repr = '<__repr__(self) failed for object at %0x>' % id(self) + + self.log_info( + 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( + self_repr, + t, + v, + tbinfo + ), + 'error' + ) + self.handle_close() + + def handle_expt(self): + self.log_info('unhandled incoming priority event', 'warning') + + def handle_read(self): + self.log_info('unhandled read event', 'warning') + + def handle_write(self): + self.log_info('unhandled write event', 'warning') + + def handle_connect(self): + self.log_info('unhandled connect event', 'warning') + + def handle_accept(self): + pair = self.accept() + if pair is not None: + self.handle_accepted(*pair) + + def handle_accepted(self, sock, addr): + sock.close() + self.log_info('unhandled accepted event', 'warning') + + def handle_close(self): + self.log_info('unhandled close event', 'warning') + self.close() + +# --------------------------------------------------------------------------- +# adds simple buffered output capability, useful for simple clients. +# [for more sophisticated usage use asynchat.async_chat] +# --------------------------------------------------------------------------- + +class dispatcher_with_send(dispatcher): + + def __init__(self, sock=None, map=None): + dispatcher.__init__(self, sock, map) + self.out_buffer = b'' + + def initiate_send(self): + num_sent = 0 + num_sent = dispatcher.send(self, self.out_buffer[:65536]) + self.out_buffer = self.out_buffer[num_sent:] + + def handle_write(self): + self.initiate_send() + + def writable(self): + return (not self.connected) or len(self.out_buffer) + + def send(self, data): + if self.debug: + self.log_info('sending %s' % repr(data)) + self.out_buffer = self.out_buffer + data + self.initiate_send() + +# --------------------------------------------------------------------------- +# used for debugging. +# --------------------------------------------------------------------------- + +def compact_traceback(): + t, v, tb = sys.exc_info() + tbinfo = [] + if not tb: # Must have a traceback + raise AssertionError("traceback does not exist") + while tb: + tbinfo.append(( + tb.tb_frame.f_code.co_filename, + tb.tb_frame.f_code.co_name, + str(tb.tb_lineno) + )) + tb = tb.tb_next + + # just to be safe + del tb + + file, function, line = tbinfo[-1] + info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) + return (file, function, line), t, v, info + +def close_all(map=None, ignore_all=False): + if map is None: + map = socket_map + for x in list(map.values()): + try: + x.close() + except OSError as x: + if x.errno == EBADF: + pass + elif not ignore_all: + raise + except _reraised_exceptions: + raise + except: + if not ignore_all: + raise + map.clear() + +# Asynchronous File I/O: +# +# After a little research (reading man pages on various unixen, and +# digging through the linux kernel), I've determined that select() +# isn't meant for doing asynchronous file i/o. +# Heartening, though - reading linux/mm/filemap.c shows that linux +# supports asynchronous read-ahead. So _MOST_ of the time, the data +# will be sitting in memory for us already when we go to read it. +# +# What other OS's (besides NT) support async file i/o? [VMS?] +# +# Regardless, this is useful for pipes, and stdin/stdout... + +if os.name == 'posix': + class file_wrapper: + # Here we override just enough to make a file + # look like a socket for the purposes of asyncore. + # The passed fd is automatically os.dup()'d + + def __init__(self, fd): + self.fd = os.dup(fd) + + def __del__(self): + if self.fd >= 0: + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) + self.close() + + def recv(self, *args): + return os.read(self.fd, *args) + + def send(self, *args): + return os.write(self.fd, *args) + + def getsockopt(self, level, optname, buflen=None): + if (level == socket.SOL_SOCKET and + optname == socket.SO_ERROR and + not buflen): + return 0 + raise NotImplementedError("Only asyncore specific behaviour " + "implemented.") + + read = recv + write = send + + def close(self): + if self.fd < 0: + return + fd = self.fd + self.fd = -1 + os.close(fd) + + def fileno(self): + return self.fd + + class file_dispatcher(dispatcher): + + def __init__(self, fd, map=None): + dispatcher.__init__(self, None, map) + self.connected = True + try: + fd = fd.fileno() + except AttributeError: + pass + self.set_file(fd) + # set it to non-blocking mode + os.set_blocking(fd, False) + + def set_file(self, fd): + self.socket = file_wrapper(fd) + self._fileno = self.socket.fileno() + self.add_channel() diff --git a/Lib/smtpd.py b/Lib/smtpd.py new file mode 100755 index 0000000..1cd004f --- /dev/null +++ b/Lib/smtpd.py @@ -0,0 +1,885 @@ +#! /usr/bin/env python3 +"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. + +Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] + +Options: + + --nosetuid + -n + This program generally tries to setuid `nobody', unless this flag is + set. The setuid call will fail if this program is not run as root (in + which case, use this flag). + + --version + -V + Print the version number and exit. + + --class classname + -c classname + Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + default. + + --size limit + -s limit + Restrict the total size of the incoming message to "limit" number of + bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. + + --smtputf8 + -u + Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. + + --debug + -d + Turn on debugging prints. + + --help + -h + Print this message and exit. + +Version: %(__version__)s + +If localhost is not given then `localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then `localhost' is used, +and if remoteport is not given, then 25 is used. +""" + +# Overview: +# +# This file implements the minimal SMTP protocol as defined in RFC 5321. It +# has a hierarchy of classes which implement the backend functionality for the +# smtpd. A number of classes are provided: +# +# SMTPServer - the base class for the backend. Raises NotImplementedError +# if you try to use it. +# +# DebuggingServer - simply prints each message it receives on stdout. +# +# PureProxy - Proxies all messages to a real smtpd which does final +# delivery. One known problem with this class is that it doesn't handle +# SMTP errors from the backend server at all. This should be fixed +# (contributions are welcome!). +# +# +# Author: Barry Warsaw +# +# TODO: +# +# - support mailbox delivery +# - alias files +# - Handle more ESMTP extensions +# - handle error codes from the backend smtpd + +import sys +import os +import errno +import getopt +import time +import socket +import collections +from warnings import warn +from email._header_value_parser import get_addr_spec, get_angle_addr + +__all__ = [ + "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", +] + +warn( + 'The smtpd module is deprecated and unmaintained. Please see aiosmtpd ' + '(https://aiosmtpd.readthedocs.io/) for the recommended replacement.', + DeprecationWarning, + stacklevel=2) + + +# These are imported after the above warning so that users get the correct +# deprecation warning. +import asyncore +import asynchat + + +program = sys.argv[0] +__version__ = 'Python SMTP proxy version 0.3' + + +class Devnull: + def write(self, msg): pass + def flush(self): pass + + +DEBUGSTREAM = Devnull() +NEWLINE = '\n' +COMMASPACE = ', ' +DATA_SIZE_DEFAULT = 33554432 + + +def usage(code, msg=''): + print(__doc__ % globals(), file=sys.stderr) + if msg: + print(msg, file=sys.stderr) + sys.exit(code) + + +class SMTPChannel(asynchat.async_chat): + COMMAND = 0 + DATA = 1 + + command_size_limit = 512 + command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) + + @property + def max_command_size_limit(self): + try: + return max(self.command_size_limits.values()) + except ValueError: + return self.command_size_limit + + def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, + map=None, enable_SMTPUTF8=False, decode_data=False): + asynchat.async_chat.__init__(self, conn, map=map) + self.smtp_server = server + self.conn = conn + self.addr = addr + self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + self._decode_data = decode_data + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") + if decode_data: + self._emptystring = '' + self._linesep = '\r\n' + self._dotsep = '.' + self._newline = NEWLINE + else: + self._emptystring = b'' + self._linesep = b'\r\n' + self._dotsep = ord(b'.') + self._newline = b'\n' + self._set_rset_state() + self.seen_greeting = '' + self.extended_smtp = False + self.command_size_limits.clear() + self.fqdn = socket.getfqdn() + try: + self.peer = conn.getpeername() + except OSError as err: + # a race condition may occur if the other end is closing + # before we can get the peername + self.close() + if err.errno != errno.ENOTCONN: + raise + return + print('Peer:', repr(self.peer), file=DEBUGSTREAM) + self.push('220 %s %s' % (self.fqdn, __version__)) + + def _set_post_data_state(self): + """Reset state variables to their post-DATA state.""" + self.smtp_state = self.COMMAND + self.mailfrom = None + self.rcpttos = [] + self.require_SMTPUTF8 = False + self.num_bytes = 0 + self.set_terminator(b'\r\n') + + def _set_rset_state(self): + """Reset all state variables except the greeting.""" + self._set_post_data_state() + self.received_data = '' + self.received_lines = [] + + + # properties for backwards-compatibility + @property + def __server(self): + warn("Access to __server attribute on SMTPChannel is deprecated, " + "use 'smtp_server' instead", DeprecationWarning, 2) + return self.smtp_server + @__server.setter + def __server(self, value): + warn("Setting __server attribute on SMTPChannel is deprecated, " + "set 'smtp_server' instead", DeprecationWarning, 2) + self.smtp_server = value + + @property + def __line(self): + warn("Access to __line attribute on SMTPChannel is deprecated, " + "use 'received_lines' instead", DeprecationWarning, 2) + return self.received_lines + @__line.setter + def __line(self, value): + warn("Setting __line attribute on SMTPChannel is deprecated, " + "set 'received_lines' instead", DeprecationWarning, 2) + self.received_lines = value + + @property + def __state(self): + warn("Access to __state attribute on SMTPChannel is deprecated, " + "use 'smtp_state' instead", DeprecationWarning, 2) + return self.smtp_state + @__state.setter + def __state(self, value): + warn("Setting __state attribute on SMTPChannel is deprecated, " + "set 'smtp_state' instead", DeprecationWarning, 2) + self.smtp_state = value + + @property + def __greeting(self): + warn("Access to __greeting attribute on SMTPChannel is deprecated, " + "use 'seen_greeting' instead", DeprecationWarning, 2) + return self.seen_greeting + @__greeting.setter + def __greeting(self, value): + warn("Setting __greeting attribute on SMTPChannel is deprecated, " + "set 'seen_greeting' instead", DeprecationWarning, 2) + self.seen_greeting = value + + @property + def __mailfrom(self): + warn("Access to __mailfrom attribute on SMTPChannel is deprecated, " + "use 'mailfrom' instead", DeprecationWarning, 2) + return self.mailfrom + @__mailfrom.setter + def __mailfrom(self, value): + warn("Setting __mailfrom attribute on SMTPChannel is deprecated, " + "set 'mailfrom' instead", DeprecationWarning, 2) + self.mailfrom = value + + @property + def __rcpttos(self): + warn("Access to __rcpttos attribute on SMTPChannel is deprecated, " + "use 'rcpttos' instead", DeprecationWarning, 2) + return self.rcpttos + @__rcpttos.setter + def __rcpttos(self, value): + warn("Setting __rcpttos attribute on SMTPChannel is deprecated, " + "set 'rcpttos' instead", DeprecationWarning, 2) + self.rcpttos = value + + @property + def __data(self): + warn("Access to __data attribute on SMTPChannel is deprecated, " + "use 'received_data' instead", DeprecationWarning, 2) + return self.received_data + @__data.setter + def __data(self, value): + warn("Setting __data attribute on SMTPChannel is deprecated, " + "set 'received_data' instead", DeprecationWarning, 2) + self.received_data = value + + @property + def __fqdn(self): + warn("Access to __fqdn attribute on SMTPChannel is deprecated, " + "use 'fqdn' instead", DeprecationWarning, 2) + return self.fqdn + @__fqdn.setter + def __fqdn(self, value): + warn("Setting __fqdn attribute on SMTPChannel is deprecated, " + "set 'fqdn' instead", DeprecationWarning, 2) + self.fqdn = value + + @property + def __peer(self): + warn("Access to __peer attribute on SMTPChannel is deprecated, " + "use 'peer' instead", DeprecationWarning, 2) + return self.peer + @__peer.setter + def __peer(self, value): + warn("Setting __peer attribute on SMTPChannel is deprecated, " + "set 'peer' instead", DeprecationWarning, 2) + self.peer = value + + @property + def __conn(self): + warn("Access to __conn attribute on SMTPChannel is deprecated, " + "use 'conn' instead", DeprecationWarning, 2) + return self.conn + @__conn.setter + def __conn(self, value): + warn("Setting __conn attribute on SMTPChannel is deprecated, " + "set 'conn' instead", DeprecationWarning, 2) + self.conn = value + + @property + def __addr(self): + warn("Access to __addr attribute on SMTPChannel is deprecated, " + "use 'addr' instead", DeprecationWarning, 2) + return self.addr + @__addr.setter + def __addr(self, value): + warn("Setting __addr attribute on SMTPChannel is deprecated, " + "set 'addr' instead", DeprecationWarning, 2) + self.addr = value + + # Overrides base class for convenience. + def push(self, msg): + asynchat.async_chat.push(self, bytes( + msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) + + # Implementation of base class abstract method + def collect_incoming_data(self, data): + limit = None + if self.smtp_state == self.COMMAND: + limit = self.max_command_size_limit + elif self.smtp_state == self.DATA: + limit = self.data_size_limit + if limit and self.num_bytes > limit: + return + elif limit: + self.num_bytes += len(data) + if self._decode_data: + self.received_lines.append(str(data, 'utf-8')) + else: + self.received_lines.append(data) + + # Implementation of base class abstract method + def found_terminator(self): + line = self._emptystring.join(self.received_lines) + print('Data:', repr(line), file=DEBUGSTREAM) + self.received_lines = [] + if self.smtp_state == self.COMMAND: + sz, self.num_bytes = self.num_bytes, 0 + if not line: + self.push('500 Error: bad syntax') + return + if not self._decode_data: + line = str(line, 'utf-8') + i = line.find(' ') + if i < 0: + command = line.upper() + arg = None + else: + command = line[:i].upper() + arg = line[i+1:].strip() + max_sz = (self.command_size_limits[command] + if self.extended_smtp else self.command_size_limit) + if sz > max_sz: + self.push('500 Error: line too long') + return + method = getattr(self, 'smtp_' + command, None) + if not method: + self.push('500 Error: command "%s" not recognized' % command) + return + method(arg) + return + else: + if self.smtp_state != self.DATA: + self.push('451 Internal confusion') + self.num_bytes = 0 + return + if self.data_size_limit and self.num_bytes > self.data_size_limit: + self.push('552 Error: Too much mail data') + self.num_bytes = 0 + return + # Remove extraneous carriage returns and de-transparency according + # to RFC 5321, Section 4.5.2. + data = [] + for text in line.split(self._linesep): + if text and text[0] == self._dotsep: + data.append(text[1:]) + else: + data.append(text) + self.received_data = self._newline.join(data) + args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) + kwargs = {} + if not self._decode_data: + kwargs = { + 'mail_options': self.mail_options, + 'rcpt_options': self.rcpt_options, + } + status = self.smtp_server.process_message(*args, **kwargs) + self._set_post_data_state() + if not status: + self.push('250 OK') + else: + self.push(status) + + # SMTP and ESMTP commands + def smtp_HELO(self, arg): + if not arg: + self.push('501 Syntax: HELO hostname') + return + # See issue #21783 for a discussion of this behavior. + if self.seen_greeting: + self.push('503 Duplicate HELO/EHLO') + return + self._set_rset_state() + self.seen_greeting = arg + self.push('250 %s' % self.fqdn) + + def smtp_EHLO(self, arg): + if not arg: + self.push('501 Syntax: EHLO hostname') + return + # See issue #21783 for a discussion of this behavior. + if self.seen_greeting: + self.push('503 Duplicate HELO/EHLO') + return + self._set_rset_state() + self.seen_greeting = arg + self.extended_smtp = True + self.push('250-%s' % self.fqdn) + if self.data_size_limit: + self.push('250-SIZE %s' % self.data_size_limit) + self.command_size_limits['MAIL'] += 26 + if not self._decode_data: + self.push('250-8BITMIME') + if self.enable_SMTPUTF8: + self.push('250-SMTPUTF8') + self.command_size_limits['MAIL'] += 10 + self.push('250 HELP') + + def smtp_NOOP(self, arg): + if arg: + self.push('501 Syntax: NOOP') + else: + self.push('250 OK') + + def smtp_QUIT(self, arg): + # args is ignored + self.push('221 Bye') + self.close_when_done() + + def _strip_command_keyword(self, keyword, arg): + keylen = len(keyword) + if arg[:keylen].upper() == keyword: + return arg[keylen:].strip() + return '' + + def _getaddr(self, arg): + if not arg: + return '', '' + if arg.lstrip().startswith('<'): + address, rest = get_angle_addr(arg) + else: + address, rest = get_addr_spec(arg) + if not address: + return address, rest + return address.addr_spec, rest + + def _getparams(self, params): + # Return params as dictionary. Return None if not all parameters + # appear to be syntactically valid according to RFC 1869. + result = {} + for param in params: + param, eq, value = param.partition('=') + if not param.isalnum() or eq and not value: + return None + result[param] = value if eq else True + return result + + def smtp_HELP(self, arg): + if arg: + extended = ' [SP ]' + lc_arg = arg.upper() + if lc_arg == 'EHLO': + self.push('250 Syntax: EHLO hostname') + elif lc_arg == 'HELO': + self.push('250 Syntax: HELO hostname') + elif lc_arg == 'MAIL': + msg = '250 Syntax: MAIL FROM:
' + if self.extended_smtp: + msg += extended + self.push(msg) + elif lc_arg == 'RCPT': + msg = '250 Syntax: RCPT TO:
' + if self.extended_smtp: + msg += extended + self.push(msg) + elif lc_arg == 'DATA': + self.push('250 Syntax: DATA') + elif lc_arg == 'RSET': + self.push('250 Syntax: RSET') + elif lc_arg == 'NOOP': + self.push('250 Syntax: NOOP') + elif lc_arg == 'QUIT': + self.push('250 Syntax: QUIT') + elif lc_arg == 'VRFY': + self.push('250 Syntax: VRFY
') + else: + self.push('501 Supported commands: EHLO HELO MAIL RCPT ' + 'DATA RSET NOOP QUIT VRFY') + else: + self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' + 'RSET NOOP QUIT VRFY') + + def smtp_VRFY(self, arg): + if arg: + address, params = self._getaddr(arg) + if address: + self.push('252 Cannot VRFY user, but will accept message ' + 'and attempt delivery') + else: + self.push('502 Could not VRFY %s' % arg) + else: + self.push('501 Syntax: VRFY
') + + def smtp_MAIL(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first') + return + print('===> MAIL', arg, file=DEBUGSTREAM) + syntaxerr = '501 Syntax: MAIL FROM:
' + if self.extended_smtp: + syntaxerr += ' [SP ]' + if arg is None: + self.push(syntaxerr) + return + arg = self._strip_command_keyword('FROM:', arg) + address, params = self._getaddr(arg) + if not address: + self.push(syntaxerr) + return + if not self.extended_smtp and params: + self.push(syntaxerr) + return + if self.mailfrom: + self.push('503 Error: nested MAIL command') + return + self.mail_options = params.upper().split() + params = self._getparams(self.mail_options) + if params is None: + self.push(syntaxerr) + return + if not self._decode_data: + body = params.pop('BODY', '7BIT') + if body not in ['7BIT', '8BITMIME']: + self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') + return + if self.enable_SMTPUTF8: + smtputf8 = params.pop('SMTPUTF8', False) + if smtputf8 is True: + self.require_SMTPUTF8 = True + elif smtputf8 is not False: + self.push('501 Error: SMTPUTF8 takes no arguments') + return + size = params.pop('SIZE', None) + if size: + if not size.isdigit(): + self.push(syntaxerr) + return + elif self.data_size_limit and int(size) > self.data_size_limit: + self.push('552 Error: message size exceeds fixed maximum message size') + return + if len(params.keys()) > 0: + self.push('555 MAIL FROM parameters not recognized or not implemented') + return + self.mailfrom = address + print('sender:', self.mailfrom, file=DEBUGSTREAM) + self.push('250 OK') + + def smtp_RCPT(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first'); + return + print('===> RCPT', arg, file=DEBUGSTREAM) + if not self.mailfrom: + self.push('503 Error: need MAIL command') + return + syntaxerr = '501 Syntax: RCPT TO:
' + if self.extended_smtp: + syntaxerr += ' [SP ]' + if arg is None: + self.push(syntaxerr) + return + arg = self._strip_command_keyword('TO:', arg) + address, params = self._getaddr(arg) + if not address: + self.push(syntaxerr) + return + if not self.extended_smtp and params: + self.push(syntaxerr) + return + self.rcpt_options = params.upper().split() + params = self._getparams(self.rcpt_options) + if params is None: + self.push(syntaxerr) + return + # XXX currently there are no options we recognize. + if len(params.keys()) > 0: + self.push('555 RCPT TO parameters not recognized or not implemented') + return + self.rcpttos.append(address) + print('recips:', self.rcpttos, file=DEBUGSTREAM) + self.push('250 OK') + + def smtp_RSET(self, arg): + if arg: + self.push('501 Syntax: RSET') + return + self._set_rset_state() + self.push('250 OK') + + def smtp_DATA(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first'); + return + if not self.rcpttos: + self.push('503 Error: need RCPT command') + return + if arg: + self.push('501 Syntax: DATA') + return + self.smtp_state = self.DATA + self.set_terminator(b'\r\n.\r\n') + self.push('354 End data with .') + + # Commands that have not been implemented + def smtp_EXPN(self, arg): + self.push('502 EXPN not implemented') + + +class SMTPServer(asyncore.dispatcher): + # SMTPChannel class to use for managing client connections + channel_class = SMTPChannel + + def __init__(self, localaddr, remoteaddr, + data_size_limit=DATA_SIZE_DEFAULT, map=None, + enable_SMTPUTF8=False, decode_data=False): + self._localaddr = localaddr + self._remoteaddr = remoteaddr + self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + self._decode_data = decode_data + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") + asyncore.dispatcher.__init__(self, map=map) + try: + gai_results = socket.getaddrinfo(*localaddr, + type=socket.SOCK_STREAM) + self.create_socket(gai_results[0][0], gai_results[0][1]) + # try to re-use a server port if possible + self.set_reuse_addr() + self.bind(localaddr) + self.listen(5) + except: + self.close() + raise + else: + print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( + self.__class__.__name__, time.ctime(time.time()), + localaddr, remoteaddr), file=DEBUGSTREAM) + + def handle_accepted(self, conn, addr): + print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) + channel = self.channel_class(self, + conn, + addr, + self.data_size_limit, + self._map, + self.enable_SMTPUTF8, + self._decode_data) + + # API for "doing something useful with the message" + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): + """Override this abstract method to handle messages from the client. + + peer is a tuple containing (ipaddr, port) of the client that made the + socket connection to our smtp port. + + mailfrom is the raw address the client claims the message is coming + from. + + rcpttos is a list of raw addresses the client wishes to deliver the + message to. + + data is a string containing the entire full text of the message, + headers (if supplied) and all. It has been `de-transparencied' + according to RFC 821, Section 4.5.2. In other words, a line + containing a `.' followed by other text has had the leading dot + removed. + + kwargs is a dictionary containing additional information. It is + empty if decode_data=True was given as init parameter, otherwise + it will contain the following keys: + 'mail_options': list of parameters to the mail command. All + elements are uppercase strings. Example: + ['BODY=8BITMIME', 'SMTPUTF8']. + 'rcpt_options': same, for the rcpt command. + + This function should return None for a normal `250 Ok' response; + otherwise, it should return the desired response string in RFC 821 + format. + + """ + raise NotImplementedError + + +class DebuggingServer(SMTPServer): + + def _print_message_content(self, peer, data): + inheaders = 1 + lines = data.splitlines() + for line in lines: + # headers first + if inheaders and not line: + peerheader = 'X-Peer: ' + peer[0] + if not isinstance(data, str): + # decoded_data=false; make header match other binary output + peerheader = repr(peerheader.encode('utf-8')) + print(peerheader) + inheaders = 0 + if not isinstance(data, str): + # Avoid spurious 'str on bytes instance' warning. + line = repr(line) + print(line) + + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): + print('---------- MESSAGE FOLLOWS ----------') + if kwargs: + if kwargs.get('mail_options'): + print('mail options: %s' % kwargs['mail_options']) + if kwargs.get('rcpt_options'): + print('rcpt options: %s\n' % kwargs['rcpt_options']) + self._print_message_content(peer, data) + print('------------ END MESSAGE ------------') + + +class PureProxy(SMTPServer): + def __init__(self, *args, **kwargs): + if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: + raise ValueError("PureProxy does not support SMTPUTF8.") + super(PureProxy, self).__init__(*args, **kwargs) + + def process_message(self, peer, mailfrom, rcpttos, data): + lines = data.split('\n') + # Look for the last header + i = 0 + for line in lines: + if not line: + break + i += 1 + lines.insert(i, 'X-Peer: %s' % peer[0]) + data = NEWLINE.join(lines) + refused = self._deliver(mailfrom, rcpttos, data) + # TBD: what to do with refused addresses? + print('we got some refusals:', refused, file=DEBUGSTREAM) + + def _deliver(self, mailfrom, rcpttos, data): + import smtplib + refused = {} + try: + s = smtplib.SMTP() + s.connect(self._remoteaddr[0], self._remoteaddr[1]) + try: + refused = s.sendmail(mailfrom, rcpttos, data) + finally: + s.quit() + except smtplib.SMTPRecipientsRefused as e: + print('got SMTPRecipientsRefused', file=DEBUGSTREAM) + refused = e.recipients + except (OSError, smtplib.SMTPException) as e: + print('got', e.__class__, file=DEBUGSTREAM) + # All recipients were refused. If the exception had an associated + # error code, use it. Otherwise,fake it with a non-triggering + # exception code. + errcode = getattr(e, 'smtp_code', -1) + errmsg = getattr(e, 'smtp_error', 'ignore') + for r in rcpttos: + refused[r] = (errcode, errmsg) + return refused + + +class Options: + setuid = True + classname = 'PureProxy' + size_limit = None + enable_SMTPUTF8 = False + + +def parseargs(): + global DEBUGSTREAM + try: + opts, args = getopt.getopt( + sys.argv[1:], 'nVhc:s:du', + ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', + 'smtputf8']) + except getopt.error as e: + usage(1, e) + + options = Options() + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-V', '--version'): + print(__version__) + sys.exit(0) + elif opt in ('-n', '--nosetuid'): + options.setuid = False + elif opt in ('-c', '--class'): + options.classname = arg + elif opt in ('-d', '--debug'): + DEBUGSTREAM = sys.stderr + elif opt in ('-u', '--smtputf8'): + options.enable_SMTPUTF8 = True + elif opt in ('-s', '--size'): + try: + int_size = int(arg) + options.size_limit = int_size + except: + print('Invalid size: ' + arg, file=sys.stderr) + sys.exit(1) + + # parse the rest of the arguments + if len(args) < 1: + localspec = 'localhost:8025' + remotespec = 'localhost:25' + elif len(args) < 2: + localspec = args[0] + remotespec = 'localhost:25' + elif len(args) < 3: + localspec = args[0] + remotespec = args[1] + else: + usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) + + # split into host/port pairs + i = localspec.find(':') + if i < 0: + usage(1, 'Bad local spec: %s' % localspec) + options.localhost = localspec[:i] + try: + options.localport = int(localspec[i+1:]) + except ValueError: + usage(1, 'Bad local port: %s' % localspec) + i = remotespec.find(':') + if i < 0: + usage(1, 'Bad remote spec: %s' % remotespec) + options.remotehost = remotespec[:i] + try: + options.remoteport = int(remotespec[i+1:]) + except ValueError: + usage(1, 'Bad remote port: %s' % remotespec) + return options + + +if __name__ == '__main__': + options = parseargs() + # Become nobody + classname = options.classname + if "." in classname: + lastdot = classname.rfind(".") + mod = __import__(classname[:lastdot], globals(), locals(), [""]) + classname = classname[lastdot+1:] + else: + import __main__ as mod + class_ = getattr(mod, classname) + proxy = class_((options.localhost, options.localport), + (options.remotehost, options.remoteport), + options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) + if options.setuid: + try: + import pwd + except ImportError: + print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) + sys.exit(1) + nobody = pwd.getpwnam('nobody')[2] + try: + os.setuid(nobody) + except PermissionError: + print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) + sys.exit(1) + try: + asyncore.loop() + except KeyboardInterrupt: + pass diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 17dda99..60c9be2 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -52,7 +52,7 @@ class saved_test_environment: resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', + 'warnings.filters', 'asyncore.socket_map', 'logging._handlers', 'logging._handlerList', 'sys.gettrace', 'sys.warnoptions', # multiprocessing.process._cleanup() may release ref @@ -160,6 +160,16 @@ class saved_test_environment: warnings.filters = saved_filters[1] warnings.filters[:] = saved_filters[2] + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + def get_shutil_archive_formats(self): shutil = self.try_get_module('shutil') # we could call get_archives_formats() but that only returns the diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py index 9788d58..c7abddc 100644 --- a/Lib/test/mock_socket.py +++ b/Lib/test/mock_socket.py @@ -1,4 +1,4 @@ -"""Mock socket module used by the smtplib tests. +"""Mock socket module used by the smtpd and smtplib tests. """ # imported for _GLOBAL_DEFAULT_TIMEOUT @@ -33,7 +33,7 @@ class MockFile: class MockSocket: - """Mock socket object used by smtplib tests. + """Mock socket object used by smtpd and smtplib tests. """ def __init__(self, family=None): global _reply_data diff --git a/Lib/test/support/_asynchat.py b/Lib/test/support/_asynchat.py deleted file mode 100644 index 941cc1d..0000000 --- a/Lib/test/support/_asynchat.py +++ /dev/null @@ -1,307 +0,0 @@ -# -*- Mode: Python; tab-width: 4 -*- -# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -r"""A class supporting chat-style (command/response) protocols. - -This class adds support for 'chat' style protocols - where one side -sends a 'command', and the other sends a response (examples would be -the common internet protocols - smtp, nntp, ftp, etc..). - -The handle_read() method looks at the input stream for the current -'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' -for multi-line output), calling self.found_terminator() on its -receipt. - -for example: -Say you build an async nntp client using this class. At the start -of the connection, you'll have self.terminator set to '\r\n', in -order to process the single-line greeting. Just before issuing a -'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST -command will be accumulated (using your own 'collect_incoming_data' -method) up to the terminator, and then control will be returned to -you - by calling your self.found_terminator() method. -""" -from test.support import _asyncore as asyncore -from collections import deque - - -class async_chat(asyncore.dispatcher): - """This is an abstract class. You must derive from this class, and add - the two methods collect_incoming_data() and found_terminator()""" - - # these are overridable defaults - - ac_in_buffer_size = 65536 - ac_out_buffer_size = 65536 - - # we don't want to enable the use of encoding by default, because that is a - # sign of an application bug that we don't want to pass silently - - use_encoding = 0 - encoding = 'latin-1' - - def __init__(self, sock=None, map=None): - # for string terminator matching - self.ac_in_buffer = b'' - - # we use a list here rather than io.BytesIO for a few reasons... - # del lst[:] is faster than bio.truncate(0) - # lst = [] is faster than bio.truncate(0) - self.incoming = [] - - # we toss the use of the "simple producer" and replace it with - # a pure deque, which the original fifo was a wrapping of - self.producer_fifo = deque() - asyncore.dispatcher.__init__(self, sock, map) - - def collect_incoming_data(self, data): - raise NotImplementedError("must be implemented in subclass") - - def _collect_incoming_data(self, data): - self.incoming.append(data) - - def _get_data(self): - d = b''.join(self.incoming) - del self.incoming[:] - return d - - def found_terminator(self): - raise NotImplementedError("must be implemented in subclass") - - def set_terminator(self, term): - """Set the input delimiter. - - Can be a fixed string of any length, an integer, or None. - """ - if isinstance(term, str) and self.use_encoding: - term = bytes(term, self.encoding) - elif isinstance(term, int) and term < 0: - raise ValueError('the number of received bytes must be positive') - self.terminator = term - - def get_terminator(self): - return self.terminator - - # grab some more data from the socket, - # throw it to the collector method, - # check for the terminator, - # if found, transition to the next state. - - def handle_read(self): - - try: - data = self.recv(self.ac_in_buffer_size) - except BlockingIOError: - return - except OSError: - self.handle_error() - return - - if isinstance(data, str) and self.use_encoding: - data = bytes(str, self.encoding) - self.ac_in_buffer = self.ac_in_buffer + data - - # Continue to search for self.terminator in self.ac_in_buffer, - # while calling self.collect_incoming_data. The while loop - # is necessary because we might read several data+terminator - # combos with a single recv(4096). - - while self.ac_in_buffer: - lb = len(self.ac_in_buffer) - terminator = self.get_terminator() - if not terminator: - # no terminator, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - elif isinstance(terminator, int): - # numeric terminator - n = terminator - if lb < n: - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - self.terminator = self.terminator - lb - else: - self.collect_incoming_data(self.ac_in_buffer[:n]) - self.ac_in_buffer = self.ac_in_buffer[n:] - self.terminator = 0 - self.found_terminator() - else: - # 3 cases: - # 1) end of buffer matches terminator exactly: - # collect data, transition - # 2) end of buffer matches some prefix: - # collect data to the prefix - # 3) end of buffer does not match any prefix: - # collect data - terminator_len = len(terminator) - index = self.ac_in_buffer.find(terminator) - if index != -1: - # we found the terminator - if index > 0: - # don't bother reporting the empty string - # (source of subtle bugs) - self.collect_incoming_data(self.ac_in_buffer[:index]) - self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] - # This does the Right Thing if the terminator - # is changed here. - self.found_terminator() - else: - # check for a prefix of the terminator - index = find_prefix_at_end(self.ac_in_buffer, terminator) - if index: - if index != lb: - # we found a prefix, collect up to the prefix - self.collect_incoming_data(self.ac_in_buffer[:-index]) - self.ac_in_buffer = self.ac_in_buffer[-index:] - break - else: - # no prefix, collect it all - self.collect_incoming_data(self.ac_in_buffer) - self.ac_in_buffer = b'' - - def handle_write(self): - self.initiate_send() - - def handle_close(self): - self.close() - - def push(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be byte-ish (%r)', - type(data)) - sabs = self.ac_out_buffer_size - if len(data) > sabs: - for i in range(0, len(data), sabs): - self.producer_fifo.append(data[i:i+sabs]) - else: - self.producer_fifo.append(data) - self.initiate_send() - - def push_with_producer(self, producer): - self.producer_fifo.append(producer) - self.initiate_send() - - def readable(self): - "predicate for inclusion in the readable for select()" - # cannot use the old predicate, it violates the claim of the - # set_terminator method. - - # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) - return 1 - - def writable(self): - "predicate for inclusion in the writable for select()" - return self.producer_fifo or (not self.connected) - - def close_when_done(self): - "automatically close this channel once the outgoing queue is empty" - self.producer_fifo.append(None) - - def initiate_send(self): - while self.producer_fifo and self.connected: - first = self.producer_fifo[0] - # handle empty string/buffer or None entry - if not first: - del self.producer_fifo[0] - if first is None: - self.handle_close() - return - - # handle classic producer behavior - obs = self.ac_out_buffer_size - try: - data = first[:obs] - except TypeError: - data = first.more() - if data: - self.producer_fifo.appendleft(data) - else: - del self.producer_fifo[0] - continue - - if isinstance(data, str) and self.use_encoding: - data = bytes(data, self.encoding) - - # send the data - try: - num_sent = self.send(data) - except OSError: - self.handle_error() - return - - if num_sent: - if num_sent < len(data) or obs < len(first): - self.producer_fifo[0] = first[num_sent:] - else: - del self.producer_fifo[0] - # we tried to send some actual data - return - - def discard_buffers(self): - # Emergencies only! - self.ac_in_buffer = b'' - del self.incoming[:] - self.producer_fifo.clear() - - -class simple_producer: - - def __init__(self, data, buffer_size=512): - self.data = data - self.buffer_size = buffer_size - - def more(self): - if len(self.data) > self.buffer_size: - result = self.data[:self.buffer_size] - self.data = self.data[self.buffer_size:] - return result - else: - result = self.data - self.data = b'' - return result - - -# Given 'haystack', see if any prefix of 'needle' is at its end. This -# assumes an exact match has already been checked. Return the number of -# characters matched. -# for example: -# f_p_a_e("qwerty\r", "\r\n") => 1 -# f_p_a_e("qwertydkjf", "\r\n") => 0 -# f_p_a_e("qwerty\r\n", "\r\n") => - -# this could maybe be made faster with a computed regex? -# [answer: no; circa Python-2.0, Jan 2001] -# new python: 28961/s -# old python: 18307/s -# re: 12820/s -# regex: 14035/s - -def find_prefix_at_end(haystack, needle): - l = len(needle) - 1 - while l and not haystack.endswith(needle[:l]): - l -= 1 - return l diff --git a/Lib/test/support/_asyncore.py b/Lib/test/support/_asyncore.py deleted file mode 100644 index 7863efa..0000000 --- a/Lib/test/support/_asyncore.py +++ /dev/null @@ -1,643 +0,0 @@ -# -*- Mode: Python -*- -# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp -# Author: Sam Rushing - -# ====================================================================== -# Copyright 1996 by Sam Rushing -# -# All Rights Reserved -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose and without fee is hereby -# granted, provided that the above copyright notice appear in all -# copies and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of Sam -# Rushing not be used in advertising or publicity pertaining to -# distribution of the software without specific, written prior -# permission. -# -# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, -# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN -# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR -# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# ====================================================================== - -"""Basic infrastructure for asynchronous socket service clients and servers. - -There are only two ways to have a program on a single processor do "more -than one thing at a time". Multi-threaded programming is the simplest and -most popular way to do it, but there is another very different technique, -that lets you have nearly all the advantages of multi-threading, without -actually using multiple threads. it's really only practical if your program -is largely I/O bound. If your program is CPU bound, then pre-emptive -scheduled threads are probably what you really need. Network servers are -rarely CPU-bound, however. - -If your operating system supports the select() system call in its I/O -library (and nearly all do), then you can use it to juggle multiple -communication channels at once; doing other work while your I/O is taking -place in the "background." Although this strategy can seem strange and -complex, especially at first, it is in many ways easier to understand and -control than multi-threaded programming. The module documented here solves -many of the difficult problems for you, making the task of building -sophisticated high-performance network servers and clients a snap. -""" - -import select -import socket -import sys -import time -import warnings - -import os -from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ - ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ - errorcode - - -_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, - EBADF}) - -try: - socket_map -except NameError: - socket_map = {} - -def _strerror(err): - try: - return os.strerror(err) - except (ValueError, OverflowError, NameError): - if err in errorcode: - return errorcode[err] - return "Unknown error %s" %err - -class ExitNow(Exception): - pass - -_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) - -def read(obj): - try: - obj.handle_read_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def write(obj): - try: - obj.handle_write_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def _exception(obj): - try: - obj.handle_expt_event() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def readwrite(obj, flags): - try: - if flags & select.POLLIN: - obj.handle_read_event() - if flags & select.POLLOUT: - obj.handle_write_event() - if flags & select.POLLPRI: - obj.handle_expt_event() - if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): - obj.handle_close() - except OSError as e: - if e.errno not in _DISCONNECTED: - obj.handle_error() - else: - obj.handle_close() - except _reraised_exceptions: - raise - except: - obj.handle_error() - -def poll(timeout=0.0, map=None): - if map is None: - map = socket_map - if map: - r = []; w = []; e = [] - for fd, obj in list(map.items()): - is_r = obj.readable() - is_w = obj.writable() - if is_r: - r.append(fd) - # accepting sockets should not be writable - if is_w and not obj.accepting: - w.append(fd) - if is_r or is_w: - e.append(fd) - if [] == r == w == e: - time.sleep(timeout) - return - - r, w, e = select.select(r, w, e, timeout) - - for fd in r: - obj = map.get(fd) - if obj is None: - continue - read(obj) - - for fd in w: - obj = map.get(fd) - if obj is None: - continue - write(obj) - - for fd in e: - obj = map.get(fd) - if obj is None: - continue - _exception(obj) - -def poll2(timeout=0.0, map=None): - # Use the poll() support added to the select module in Python 2.0 - if map is None: - map = socket_map - if timeout is not None: - # timeout is in milliseconds - timeout = int(timeout*1000) - pollster = select.poll() - if map: - for fd, obj in list(map.items()): - flags = 0 - if obj.readable(): - flags |= select.POLLIN | select.POLLPRI - # accepting sockets should not be writable - if obj.writable() and not obj.accepting: - flags |= select.POLLOUT - if flags: - pollster.register(fd, flags) - - r = pollster.poll(timeout) - for fd, flags in r: - obj = map.get(fd) - if obj is None: - continue - readwrite(obj, flags) - -poll3 = poll2 # Alias for backward compatibility - -def loop(timeout=30.0, use_poll=False, map=None, count=None): - if map is None: - map = socket_map - - if use_poll and hasattr(select, 'poll'): - poll_fun = poll2 - else: - poll_fun = poll - - if count is None: - while map: - poll_fun(timeout, map) - - else: - while map and count > 0: - poll_fun(timeout, map) - count = count - 1 - -class dispatcher: - - debug = False - connected = False - accepting = False - connecting = False - closing = False - addr = None - ignore_log_types = frozenset({'warning'}) - - def __init__(self, sock=None, map=None): - if map is None: - self._map = socket_map - else: - self._map = map - - self._fileno = None - - if sock: - # Set to nonblocking just to make sure for cases where we - # get a socket from a blocking source. - sock.setblocking(False) - self.set_socket(sock, map) - self.connected = True - # The constructor no longer requires that the socket - # passed be connected. - try: - self.addr = sock.getpeername() - except OSError as err: - if err.errno in (ENOTCONN, EINVAL): - # To handle the case where we got an unconnected - # socket. - self.connected = False - else: - # The socket is broken in some unknown way, alert - # the user and remove it from the map (to prevent - # polling of broken sockets). - self.del_channel(map) - raise - else: - self.socket = None - - def __repr__(self): - status = [self.__class__.__module__+"."+self.__class__.__qualname__] - if self.accepting and self.addr: - status.append('listening') - elif self.connected: - status.append('connected') - if self.addr is not None: - try: - status.append('%s:%d' % self.addr) - except TypeError: - status.append(repr(self.addr)) - return '<%s at %#x>' % (' '.join(status), id(self)) - - def add_channel(self, map=None): - #self.log_info('adding channel %s' % self) - if map is None: - map = self._map - map[self._fileno] = self - - def del_channel(self, map=None): - fd = self._fileno - if map is None: - map = self._map - if fd in map: - #self.log_info('closing channel %d:%s' % (fd, self)) - del map[fd] - self._fileno = None - - def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): - self.family_and_type = family, type - sock = socket.socket(family, type) - sock.setblocking(False) - self.set_socket(sock) - - def set_socket(self, sock, map=None): - self.socket = sock - self._fileno = sock.fileno() - self.add_channel(map) - - def set_reuse_addr(self): - # try to re-use a server port if possible - try: - self.socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, - self.socket.getsockopt(socket.SOL_SOCKET, - socket.SO_REUSEADDR) | 1 - ) - except OSError: - pass - - # ================================================== - # predicates for select() - # these are used as filters for the lists of sockets - # to pass to select(). - # ================================================== - - def readable(self): - return True - - def writable(self): - return True - - # ================================================== - # socket object methods. - # ================================================== - - def listen(self, num): - self.accepting = True - if os.name == 'nt' and num > 5: - num = 5 - return self.socket.listen(num) - - def bind(self, addr): - self.addr = addr - return self.socket.bind(addr) - - def connect(self, address): - self.connected = False - self.connecting = True - err = self.socket.connect_ex(address) - if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ - or err == EINVAL and os.name == 'nt': - self.addr = address - return - if err in (0, EISCONN): - self.addr = address - self.handle_connect_event() - else: - raise OSError(err, errorcode[err]) - - def accept(self): - # XXX can return either an address pair or None - try: - conn, addr = self.socket.accept() - except TypeError: - return None - except OSError as why: - if why.errno in (EWOULDBLOCK, ECONNABORTED, EAGAIN): - return None - else: - raise - else: - return conn, addr - - def send(self, data): - try: - result = self.socket.send(data) - return result - except OSError as why: - if why.errno == EWOULDBLOCK: - return 0 - elif why.errno in _DISCONNECTED: - self.handle_close() - return 0 - else: - raise - - def recv(self, buffer_size): - try: - data = self.socket.recv(buffer_size) - if not data: - # a closed connection is indicated by signaling - # a read condition, and having recv() return 0. - self.handle_close() - return b'' - else: - return data - except OSError as why: - # winsock sometimes raises ENOTCONN - if why.errno in _DISCONNECTED: - self.handle_close() - return b'' - else: - raise - - def close(self): - self.connected = False - self.accepting = False - self.connecting = False - self.del_channel() - if self.socket is not None: - try: - self.socket.close() - except OSError as why: - if why.errno not in (ENOTCONN, EBADF): - raise - - # log and log_info may be overridden to provide more sophisticated - # logging and warning methods. In general, log is for 'hit' logging - # and 'log_info' is for informational, warning and error logging. - - def log(self, message): - sys.stderr.write('log: %s\n' % str(message)) - - def log_info(self, message, type='info'): - if type not in self.ignore_log_types: - print('%s: %s' % (type, message)) - - def handle_read_event(self): - if self.accepting: - # accepting sockets are never connected, they "spawn" new - # sockets that are connected - self.handle_accept() - elif not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_read() - else: - self.handle_read() - - def handle_connect_event(self): - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise OSError(err, _strerror(err)) - self.handle_connect() - self.connected = True - self.connecting = False - - def handle_write_event(self): - if self.accepting: - # Accepting sockets shouldn't get a write event. - # We will pretend it didn't happen. - return - - if not self.connected: - if self.connecting: - self.handle_connect_event() - self.handle_write() - - def handle_expt_event(self): - # handle_expt_event() is called if there might be an error on the - # socket, or if there is OOB data - # check for the error condition first - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - # we can get here when select.select() says that there is an - # exceptional condition on the socket - # since there is an error, we'll go ahead and close the socket - # like we would in a subclassed handle_read() that received no - # data - self.handle_close() - else: - self.handle_expt() - - def handle_error(self): - nil, t, v, tbinfo = compact_traceback() - - # sometimes a user repr method will crash. - try: - self_repr = repr(self) - except: - self_repr = '<__repr__(self) failed for object at %0x>' % id(self) - - self.log_info( - 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( - self_repr, - t, - v, - tbinfo - ), - 'error' - ) - self.handle_close() - - def handle_expt(self): - self.log_info('unhandled incoming priority event', 'warning') - - def handle_read(self): - self.log_info('unhandled read event', 'warning') - - def handle_write(self): - self.log_info('unhandled write event', 'warning') - - def handle_connect(self): - self.log_info('unhandled connect event', 'warning') - - def handle_accept(self): - pair = self.accept() - if pair is not None: - self.handle_accepted(*pair) - - def handle_accepted(self, sock, addr): - sock.close() - self.log_info('unhandled accepted event', 'warning') - - def handle_close(self): - self.log_info('unhandled close event', 'warning') - self.close() - -# --------------------------------------------------------------------------- -# adds simple buffered output capability, useful for simple clients. -# [for more sophisticated usage use asynchat.async_chat] -# --------------------------------------------------------------------------- - -class dispatcher_with_send(dispatcher): - - def __init__(self, sock=None, map=None): - dispatcher.__init__(self, sock, map) - self.out_buffer = b'' - - def initiate_send(self): - num_sent = 0 - num_sent = dispatcher.send(self, self.out_buffer[:65536]) - self.out_buffer = self.out_buffer[num_sent:] - - def handle_write(self): - self.initiate_send() - - def writable(self): - return (not self.connected) or len(self.out_buffer) - - def send(self, data): - if self.debug: - self.log_info('sending %s' % repr(data)) - self.out_buffer = self.out_buffer + data - self.initiate_send() - -# --------------------------------------------------------------------------- -# used for debugging. -# --------------------------------------------------------------------------- - -def compact_traceback(): - t, v, tb = sys.exc_info() - tbinfo = [] - if not tb: # Must have a traceback - raise AssertionError("traceback does not exist") - while tb: - tbinfo.append(( - tb.tb_frame.f_code.co_filename, - tb.tb_frame.f_code.co_name, - str(tb.tb_lineno) - )) - tb = tb.tb_next - - # just to be safe - del tb - - file, function, line = tbinfo[-1] - info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) - return (file, function, line), t, v, info - -def close_all(map=None, ignore_all=False): - if map is None: - map = socket_map - for x in list(map.values()): - try: - x.close() - except OSError as x: - if x.errno == EBADF: - pass - elif not ignore_all: - raise - except _reraised_exceptions: - raise - except: - if not ignore_all: - raise - map.clear() - -# Asynchronous File I/O: -# -# After a little research (reading man pages on various unixen, and -# digging through the linux kernel), I've determined that select() -# isn't meant for doing asynchronous file i/o. -# Heartening, though - reading linux/mm/filemap.c shows that linux -# supports asynchronous read-ahead. So _MOST_ of the time, the data -# will be sitting in memory for us already when we go to read it. -# -# What other OS's (besides NT) support async file i/o? [VMS?] -# -# Regardless, this is useful for pipes, and stdin/stdout... - -if os.name == 'posix': - class file_wrapper: - # Here we override just enough to make a file - # look like a socket for the purposes of asyncore. - # The passed fd is automatically os.dup()'d - - def __init__(self, fd): - self.fd = os.dup(fd) - - def __del__(self): - if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning, - source=self) - self.close() - - def recv(self, *args): - return os.read(self.fd, *args) - - def send(self, *args): - return os.write(self.fd, *args) - - def getsockopt(self, level, optname, buflen=None): - if (level == socket.SOL_SOCKET and - optname == socket.SO_ERROR and - not buflen): - return 0 - raise NotImplementedError("Only asyncore specific behaviour " - "implemented.") - - read = recv - write = send - - def close(self): - if self.fd < 0: - return - fd = self.fd - self.fd = -1 - os.close(fd) - - def fileno(self): - return self.fd - - class file_dispatcher(dispatcher): - - def __init__(self, fd, map=None): - dispatcher.__init__(self, None, map) - self.connected = True - try: - fd = fd.fileno() - except AttributeError: - pass - self.set_file(fd) - # set it to non-blocking mode - os.set_blocking(fd, False) - - def set_file(self, fd): - self.socket = file_wrapper(fd) - self._fileno = self.socket.fileno() - self.add_channel() diff --git a/Lib/test/support/_smtpd.py b/Lib/test/support/_smtpd.py deleted file mode 100755 index 0e37d08..0000000 --- a/Lib/test/support/_smtpd.py +++ /dev/null @@ -1,754 +0,0 @@ -#! /usr/bin/env python3 -"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. - -Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] - -Options: - - --nosetuid - -n - This program generally tries to setuid `nobody', unless this flag is - set. The setuid call will fail if this program is not run as root (in - which case, use this flag). - - --version - -V - Print the version number and exit. - - --class classname - -c classname - Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by - default. - - --size limit - -s limit - Restrict the total size of the incoming message to "limit" number of - bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. - - --smtputf8 - -u - Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. - - --debug - -d - Turn on debugging prints. - - --help - -h - Print this message and exit. - -Version: %(__version__)s - -If localhost is not given then `localhost' is used, and if localport is not -given then 8025 is used. If remotehost is not given then `localhost' is used, -and if remoteport is not given, then 25 is used. -""" - -# Overview: -# -# This file implements the minimal SMTP protocol as defined in RFC 5321. It -# has a hierarchy of classes which implement the backend functionality for the -# smtpd. A number of classes are provided: -# -# SMTPServer - the base class for the backend. Raises NotImplementedError -# if you try to use it. -# -# DebuggingServer - simply prints each message it receives on stdout. -# -# PureProxy - Proxies all messages to a real smtpd which does final -# delivery. One known problem with this class is that it doesn't handle -# SMTP errors from the backend server at all. This should be fixed -# (contributions are welcome!). -# -# -# Author: Barry Warsaw -# -# TODO: -# -# - support mailbox delivery -# - alias files -# - Handle more ESMTP extensions -# - handle error codes from the backend smtpd - -import sys -import os -import errno -import getopt -import time -import socket -import collections -from warnings import warn -from email._header_value_parser import get_addr_spec, get_angle_addr - -from test.support import _asyncore as asyncore -from test.support import _asynchat as asynchat - -__all__ = [ - "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", -] - - -program = sys.argv[0] -__version__ = 'Python SMTP proxy version 0.3' - - -class Devnull: - def write(self, msg): pass - def flush(self): pass - - -DEBUGSTREAM = Devnull() -NEWLINE = '\n' -COMMASPACE = ', ' -DATA_SIZE_DEFAULT = 33554432 - - -def usage(code, msg=''): - print(__doc__ % globals(), file=sys.stderr) - if msg: - print(msg, file=sys.stderr) - sys.exit(code) - - -class SMTPChannel(asynchat.async_chat): - COMMAND = 0 - DATA = 1 - - command_size_limit = 512 - command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) - - @property - def max_command_size_limit(self): - try: - return max(self.command_size_limits.values()) - except ValueError: - return self.command_size_limit - - def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, - map=None, enable_SMTPUTF8=False, decode_data=False): - asynchat.async_chat.__init__(self, conn, map=map) - self.smtp_server = server - self.conn = conn - self.addr = addr - self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - self._decode_data = decode_data - if enable_SMTPUTF8 and decode_data: - raise ValueError("decode_data and enable_SMTPUTF8 cannot" - " be set to True at the same time") - if decode_data: - self._emptystring = '' - self._linesep = '\r\n' - self._dotsep = '.' - self._newline = NEWLINE - else: - self._emptystring = b'' - self._linesep = b'\r\n' - self._dotsep = ord(b'.') - self._newline = b'\n' - self._set_rset_state() - self.seen_greeting = '' - self.extended_smtp = False - self.command_size_limits.clear() - self.fqdn = socket.getfqdn() - try: - self.peer = conn.getpeername() - except OSError as err: - # a race condition may occur if the other end is closing - # before we can get the peername - self.close() - if err.errno != errno.ENOTCONN: - raise - return - print('Peer:', repr(self.peer), file=DEBUGSTREAM) - self.push('220 %s %s' % (self.fqdn, __version__)) - - def _set_post_data_state(self): - """Reset state variables to their post-DATA state.""" - self.smtp_state = self.COMMAND - self.mailfrom = None - self.rcpttos = [] - self.require_SMTPUTF8 = False - self.num_bytes = 0 - self.set_terminator(b'\r\n') - - def _set_rset_state(self): - """Reset all state variables except the greeting.""" - self._set_post_data_state() - self.received_data = '' - self.received_lines = [] - - - # Overrides base class for convenience. - def push(self, msg): - asynchat.async_chat.push(self, bytes( - msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) - - # Implementation of base class abstract method - def collect_incoming_data(self, data): - limit = None - if self.smtp_state == self.COMMAND: - limit = self.max_command_size_limit - elif self.smtp_state == self.DATA: - limit = self.data_size_limit - if limit and self.num_bytes > limit: - return - elif limit: - self.num_bytes += len(data) - if self._decode_data: - self.received_lines.append(str(data, 'utf-8')) - else: - self.received_lines.append(data) - - # Implementation of base class abstract method - def found_terminator(self): - line = self._emptystring.join(self.received_lines) - print('Data:', repr(line), file=DEBUGSTREAM) - self.received_lines = [] - if self.smtp_state == self.COMMAND: - sz, self.num_bytes = self.num_bytes, 0 - if not line: - self.push('500 Error: bad syntax') - return - if not self._decode_data: - line = str(line, 'utf-8') - i = line.find(' ') - if i < 0: - command = line.upper() - arg = None - else: - command = line[:i].upper() - arg = line[i+1:].strip() - max_sz = (self.command_size_limits[command] - if self.extended_smtp else self.command_size_limit) - if sz > max_sz: - self.push('500 Error: line too long') - return - method = getattr(self, 'smtp_' + command, None) - if not method: - self.push('500 Error: command "%s" not recognized' % command) - return - method(arg) - return - else: - if self.smtp_state != self.DATA: - self.push('451 Internal confusion') - self.num_bytes = 0 - return - if self.data_size_limit and self.num_bytes > self.data_size_limit: - self.push('552 Error: Too much mail data') - self.num_bytes = 0 - return - # Remove extraneous carriage returns and de-transparency according - # to RFC 5321, Section 4.5.2. - data = [] - for text in line.split(self._linesep): - if text and text[0] == self._dotsep: - data.append(text[1:]) - else: - data.append(text) - self.received_data = self._newline.join(data) - args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) - kwargs = {} - if not self._decode_data: - kwargs = { - 'mail_options': self.mail_options, - 'rcpt_options': self.rcpt_options, - } - status = self.smtp_server.process_message(*args, **kwargs) - self._set_post_data_state() - if not status: - self.push('250 OK') - else: - self.push(status) - - # SMTP and ESMTP commands - def smtp_HELO(self, arg): - if not arg: - self.push('501 Syntax: HELO hostname') - return - # See issue #21783 for a discussion of this behavior. - if self.seen_greeting: - self.push('503 Duplicate HELO/EHLO') - return - self._set_rset_state() - self.seen_greeting = arg - self.push('250 %s' % self.fqdn) - - def smtp_EHLO(self, arg): - if not arg: - self.push('501 Syntax: EHLO hostname') - return - # See issue #21783 for a discussion of this behavior. - if self.seen_greeting: - self.push('503 Duplicate HELO/EHLO') - return - self._set_rset_state() - self.seen_greeting = arg - self.extended_smtp = True - self.push('250-%s' % self.fqdn) - if self.data_size_limit: - self.push('250-SIZE %s' % self.data_size_limit) - self.command_size_limits['MAIL'] += 26 - if not self._decode_data: - self.push('250-8BITMIME') - if self.enable_SMTPUTF8: - self.push('250-SMTPUTF8') - self.command_size_limits['MAIL'] += 10 - self.push('250 HELP') - - def smtp_NOOP(self, arg): - if arg: - self.push('501 Syntax: NOOP') - else: - self.push('250 OK') - - def smtp_QUIT(self, arg): - # args is ignored - self.push('221 Bye') - self.close_when_done() - - def _strip_command_keyword(self, keyword, arg): - keylen = len(keyword) - if arg[:keylen].upper() == keyword: - return arg[keylen:].strip() - return '' - - def _getaddr(self, arg): - if not arg: - return '', '' - if arg.lstrip().startswith('<'): - address, rest = get_angle_addr(arg) - else: - address, rest = get_addr_spec(arg) - if not address: - return address, rest - return address.addr_spec, rest - - def _getparams(self, params): - # Return params as dictionary. Return None if not all parameters - # appear to be syntactically valid according to RFC 1869. - result = {} - for param in params: - param, eq, value = param.partition('=') - if not param.isalnum() or eq and not value: - return None - result[param] = value if eq else True - return result - - def smtp_HELP(self, arg): - if arg: - extended = ' [SP ]' - lc_arg = arg.upper() - if lc_arg == 'EHLO': - self.push('250 Syntax: EHLO hostname') - elif lc_arg == 'HELO': - self.push('250 Syntax: HELO hostname') - elif lc_arg == 'MAIL': - msg = '250 Syntax: MAIL FROM:
' - if self.extended_smtp: - msg += extended - self.push(msg) - elif lc_arg == 'RCPT': - msg = '250 Syntax: RCPT TO:
' - if self.extended_smtp: - msg += extended - self.push(msg) - elif lc_arg == 'DATA': - self.push('250 Syntax: DATA') - elif lc_arg == 'RSET': - self.push('250 Syntax: RSET') - elif lc_arg == 'NOOP': - self.push('250 Syntax: NOOP') - elif lc_arg == 'QUIT': - self.push('250 Syntax: QUIT') - elif lc_arg == 'VRFY': - self.push('250 Syntax: VRFY
') - else: - self.push('501 Supported commands: EHLO HELO MAIL RCPT ' - 'DATA RSET NOOP QUIT VRFY') - else: - self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' - 'RSET NOOP QUIT VRFY') - - def smtp_VRFY(self, arg): - if arg: - address, params = self._getaddr(arg) - if address: - self.push('252 Cannot VRFY user, but will accept message ' - 'and attempt delivery') - else: - self.push('502 Could not VRFY %s' % arg) - else: - self.push('501 Syntax: VRFY
') - - def smtp_MAIL(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first') - return - print('===> MAIL', arg, file=DEBUGSTREAM) - syntaxerr = '501 Syntax: MAIL FROM:
' - if self.extended_smtp: - syntaxerr += ' [SP ]' - if arg is None: - self.push(syntaxerr) - return - arg = self._strip_command_keyword('FROM:', arg) - address, params = self._getaddr(arg) - if not address: - self.push(syntaxerr) - return - if not self.extended_smtp and params: - self.push(syntaxerr) - return - if self.mailfrom: - self.push('503 Error: nested MAIL command') - return - self.mail_options = params.upper().split() - params = self._getparams(self.mail_options) - if params is None: - self.push(syntaxerr) - return - if not self._decode_data: - body = params.pop('BODY', '7BIT') - if body not in ['7BIT', '8BITMIME']: - self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') - return - if self.enable_SMTPUTF8: - smtputf8 = params.pop('SMTPUTF8', False) - if smtputf8 is True: - self.require_SMTPUTF8 = True - elif smtputf8 is not False: - self.push('501 Error: SMTPUTF8 takes no arguments') - return - size = params.pop('SIZE', None) - if size: - if not size.isdigit(): - self.push(syntaxerr) - return - elif self.data_size_limit and int(size) > self.data_size_limit: - self.push('552 Error: message size exceeds fixed maximum message size') - return - if len(params.keys()) > 0: - self.push('555 MAIL FROM parameters not recognized or not implemented') - return - self.mailfrom = address - print('sender:', self.mailfrom, file=DEBUGSTREAM) - self.push('250 OK') - - def smtp_RCPT(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first'); - return - print('===> RCPT', arg, file=DEBUGSTREAM) - if not self.mailfrom: - self.push('503 Error: need MAIL command') - return - syntaxerr = '501 Syntax: RCPT TO:
' - if self.extended_smtp: - syntaxerr += ' [SP ]' - if arg is None: - self.push(syntaxerr) - return - arg = self._strip_command_keyword('TO:', arg) - address, params = self._getaddr(arg) - if not address: - self.push(syntaxerr) - return - if not self.extended_smtp and params: - self.push(syntaxerr) - return - self.rcpt_options = params.upper().split() - params = self._getparams(self.rcpt_options) - if params is None: - self.push(syntaxerr) - return - # XXX currently there are no options we recognize. - if len(params.keys()) > 0: - self.push('555 RCPT TO parameters not recognized or not implemented') - return - self.rcpttos.append(address) - print('recips:', self.rcpttos, file=DEBUGSTREAM) - self.push('250 OK') - - def smtp_RSET(self, arg): - if arg: - self.push('501 Syntax: RSET') - return - self._set_rset_state() - self.push('250 OK') - - def smtp_DATA(self, arg): - if not self.seen_greeting: - self.push('503 Error: send HELO first'); - return - if not self.rcpttos: - self.push('503 Error: need RCPT command') - return - if arg: - self.push('501 Syntax: DATA') - return - self.smtp_state = self.DATA - self.set_terminator(b'\r\n.\r\n') - self.push('354 End data with .') - - # Commands that have not been implemented - def smtp_EXPN(self, arg): - self.push('502 EXPN not implemented') - - -class SMTPServer(asyncore.dispatcher): - # SMTPChannel class to use for managing client connections - channel_class = SMTPChannel - - def __init__(self, localaddr, remoteaddr, - data_size_limit=DATA_SIZE_DEFAULT, map=None, - enable_SMTPUTF8=False, decode_data=False): - self._localaddr = localaddr - self._remoteaddr = remoteaddr - self.data_size_limit = data_size_limit - self.enable_SMTPUTF8 = enable_SMTPUTF8 - self._decode_data = decode_data - if enable_SMTPUTF8 and decode_data: - raise ValueError("decode_data and enable_SMTPUTF8 cannot" - " be set to True at the same time") - asyncore.dispatcher.__init__(self, map=map) - try: - gai_results = socket.getaddrinfo(*localaddr, - type=socket.SOCK_STREAM) - self.create_socket(gai_results[0][0], gai_results[0][1]) - # try to re-use a server port if possible - self.set_reuse_addr() - self.bind(localaddr) - self.listen(5) - except: - self.close() - raise - else: - print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( - self.__class__.__name__, time.ctime(time.time()), - localaddr, remoteaddr), file=DEBUGSTREAM) - - def handle_accepted(self, conn, addr): - print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) - channel = self.channel_class(self, - conn, - addr, - self.data_size_limit, - self._map, - self.enable_SMTPUTF8, - self._decode_data) - - # API for "doing something useful with the message" - def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): - """Override this abstract method to handle messages from the client. - - peer is a tuple containing (ipaddr, port) of the client that made the - socket connection to our smtp port. - - mailfrom is the raw address the client claims the message is coming - from. - - rcpttos is a list of raw addresses the client wishes to deliver the - message to. - - data is a string containing the entire full text of the message, - headers (if supplied) and all. It has been `de-transparencied' - according to RFC 821, Section 4.5.2. In other words, a line - containing a `.' followed by other text has had the leading dot - removed. - - kwargs is a dictionary containing additional information. It is - empty if decode_data=True was given as init parameter, otherwise - it will contain the following keys: - 'mail_options': list of parameters to the mail command. All - elements are uppercase strings. Example: - ['BODY=8BITMIME', 'SMTPUTF8']. - 'rcpt_options': same, for the rcpt command. - - This function should return None for a normal `250 Ok' response; - otherwise, it should return the desired response string in RFC 821 - format. - - """ - raise NotImplementedError - - -class DebuggingServer(SMTPServer): - - def _print_message_content(self, peer, data): - inheaders = 1 - lines = data.splitlines() - for line in lines: - # headers first - if inheaders and not line: - peerheader = 'X-Peer: ' + peer[0] - if not isinstance(data, str): - # decoded_data=false; make header match other binary output - peerheader = repr(peerheader.encode('utf-8')) - print(peerheader) - inheaders = 0 - if not isinstance(data, str): - # Avoid spurious 'str on bytes instance' warning. - line = repr(line) - print(line) - - def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): - print('---------- MESSAGE FOLLOWS ----------') - if kwargs: - if kwargs.get('mail_options'): - print('mail options: %s' % kwargs['mail_options']) - if kwargs.get('rcpt_options'): - print('rcpt options: %s\n' % kwargs['rcpt_options']) - self._print_message_content(peer, data) - print('------------ END MESSAGE ------------') - - -class PureProxy(SMTPServer): - def __init__(self, *args, **kwargs): - if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: - raise ValueError("PureProxy does not support SMTPUTF8.") - super(PureProxy, self).__init__(*args, **kwargs) - - def process_message(self, peer, mailfrom, rcpttos, data): - lines = data.split('\n') - # Look for the last header - i = 0 - for line in lines: - if not line: - break - i += 1 - lines.insert(i, 'X-Peer: %s' % peer[0]) - data = NEWLINE.join(lines) - refused = self._deliver(mailfrom, rcpttos, data) - # TBD: what to do with refused addresses? - print('we got some refusals:', refused, file=DEBUGSTREAM) - - def _deliver(self, mailfrom, rcpttos, data): - import smtplib - refused = {} - try: - s = smtplib.SMTP() - s.connect(self._remoteaddr[0], self._remoteaddr[1]) - try: - refused = s.sendmail(mailfrom, rcpttos, data) - finally: - s.quit() - except smtplib.SMTPRecipientsRefused as e: - print('got SMTPRecipientsRefused', file=DEBUGSTREAM) - refused = e.recipients - except (OSError, smtplib.SMTPException) as e: - print('got', e.__class__, file=DEBUGSTREAM) - # All recipients were refused. If the exception had an associated - # error code, use it. Otherwise,fake it with a non-triggering - # exception code. - errcode = getattr(e, 'smtp_code', -1) - errmsg = getattr(e, 'smtp_error', 'ignore') - for r in rcpttos: - refused[r] = (errcode, errmsg) - return refused - - -class Options: - setuid = True - classname = 'PureProxy' - size_limit = None - enable_SMTPUTF8 = False - - -def parseargs(): - global DEBUGSTREAM - try: - opts, args = getopt.getopt( - sys.argv[1:], 'nVhc:s:du', - ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', - 'smtputf8']) - except getopt.error as e: - usage(1, e) - - options = Options() - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-V', '--version'): - print(__version__) - sys.exit(0) - elif opt in ('-n', '--nosetuid'): - options.setuid = False - elif opt in ('-c', '--class'): - options.classname = arg - elif opt in ('-d', '--debug'): - DEBUGSTREAM = sys.stderr - elif opt in ('-u', '--smtputf8'): - options.enable_SMTPUTF8 = True - elif opt in ('-s', '--size'): - try: - int_size = int(arg) - options.size_limit = int_size - except: - print('Invalid size: ' + arg, file=sys.stderr) - sys.exit(1) - - # parse the rest of the arguments - if len(args) < 1: - localspec = 'localhost:8025' - remotespec = 'localhost:25' - elif len(args) < 2: - localspec = args[0] - remotespec = 'localhost:25' - elif len(args) < 3: - localspec = args[0] - remotespec = args[1] - else: - usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) - - # split into host/port pairs - i = localspec.find(':') - if i < 0: - usage(1, 'Bad local spec: %s' % localspec) - options.localhost = localspec[:i] - try: - options.localport = int(localspec[i+1:]) - except ValueError: - usage(1, 'Bad local port: %s' % localspec) - i = remotespec.find(':') - if i < 0: - usage(1, 'Bad remote spec: %s' % remotespec) - options.remotehost = remotespec[:i] - try: - options.remoteport = int(remotespec[i+1:]) - except ValueError: - usage(1, 'Bad remote port: %s' % remotespec) - return options - - -if __name__ == '__main__': - options = parseargs() - # Become nobody - classname = options.classname - if "." in classname: - lastdot = classname.rfind(".") - mod = __import__(classname[:lastdot], globals(), locals(), [""]) - classname = classname[lastdot+1:] - else: - import __main__ as mod - class_ = getattr(mod, classname) - proxy = class_((options.localhost, options.localport), - (options.remotehost, options.remoteport), - options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) - if options.setuid: - try: - import pwd - except ImportError: - print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) - sys.exit(1) - nobody = pwd.getpwnam('nobody')[2] - try: - os.setuid(nobody) - except PermissionError: - print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) - sys.exit(1) - try: - asyncore.loop() - except KeyboardInterrupt: - pass diff --git a/Lib/test/test_asynchat.py b/Lib/test/test_asynchat.py new file mode 100644 index 0000000..973ac1f --- /dev/null +++ b/Lib/test/test_asynchat.py @@ -0,0 +1,292 @@ +# test asynchat + +from test import support +from test.support import socket_helper +from test.support import threading_helper + +import errno +import socket +import sys +import threading +import time +import unittest +import unittest.mock + +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asynchat + import asyncore + +HOST = socket_helper.HOST +SERVER_QUIT = b'QUIT\n' + + +class echo_server(threading.Thread): + # parameter to determine the number of bytes passed back to the + # client each send + chunk_size = 1 + + def __init__(self, event): + threading.Thread.__init__(self) + self.event = event + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = socket_helper.bind_port(self.sock) + # This will be set if the client wants us to wait before echoing + # data back. + self.start_resend_event = None + + def run(self): + self.sock.listen() + self.event.set() + conn, client = self.sock.accept() + self.buffer = b"" + # collect data until quit message is seen + while SERVER_QUIT not in self.buffer: + data = conn.recv(1) + if not data: + break + self.buffer = self.buffer + data + + # remove the SERVER_QUIT message + self.buffer = self.buffer.replace(SERVER_QUIT, b'') + + if self.start_resend_event: + self.start_resend_event.wait() + + # re-send entire set of collected data + try: + # this may fail on some tests, such as test_close_when_done, + # since the client closes the channel when it's done sending + while self.buffer: + n = conn.send(self.buffer[:self.chunk_size]) + time.sleep(0.001) + self.buffer = self.buffer[n:] + except: + pass + + conn.close() + self.sock.close() + +class echo_client(asynchat.async_chat): + + def __init__(self, terminator, server_port): + asynchat.async_chat.__init__(self) + self.contents = [] + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.connect((HOST, server_port)) + self.set_terminator(terminator) + self.buffer = b"" + + def handle_connect(self): + pass + + if sys.platform == 'darwin': + # select.poll returns a select.POLLHUP at the end of the tests + # on darwin, so just ignore it + def handle_expt(self): + pass + + def collect_incoming_data(self, data): + self.buffer += data + + def found_terminator(self): + self.contents.append(self.buffer) + self.buffer = b"" + +def start_echo_server(): + event = threading.Event() + s = echo_server(event) + s.start() + event.wait() + event.clear() + time.sleep(0.01) # Give server time to start accepting. + return s, event + + +class TestAsynchat(unittest.TestCase): + usepoll = False + + def setUp(self): + self._threads = threading_helper.threading_setup() + + def tearDown(self): + threading_helper.threading_cleanup(*self._threads) + + def line_terminator_check(self, term, server_chunk): + event = threading.Event() + s = echo_server(event) + s.chunk_size = server_chunk + s.start() + event.wait() + event.clear() + time.sleep(0.01) # Give server time to start accepting. + c = echo_client(term, s.port) + c.push(b"hello ") + c.push(b"world" + term) + c.push(b"I'm not dead yet!" + term) + c.push(SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) + + # the line terminator tests below check receiving variously-sized + # chunks back from the server in order to exercise all branches of + # async_chat.handle_read + + def test_line_terminator1(self): + # test one-character terminator + for l in (1, 2, 3): + self.line_terminator_check(b'\n', l) + + def test_line_terminator2(self): + # test two-character terminator + for l in (1, 2, 3): + self.line_terminator_check(b'\r\n', l) + + def test_line_terminator3(self): + # test three-character terminator + for l in (1, 2, 3): + self.line_terminator_check(b'qqq', l) + + def numeric_terminator_check(self, termlen): + # Try reading a fixed number of bytes + s, event = start_echo_server() + c = echo_client(termlen, s.port) + data = b"hello world, I'm not dead yet!\n" + c.push(data) + c.push(SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, [data[:termlen]]) + + def test_numeric_terminator1(self): + # check that ints & longs both work (since type is + # explicitly checked in async_chat.handle_read) + self.numeric_terminator_check(1) + + def test_numeric_terminator2(self): + self.numeric_terminator_check(6) + + def test_none_terminator(self): + # Try reading a fixed number of bytes + s, event = start_echo_server() + c = echo_client(None, s.port) + data = b"hello world, I'm not dead yet!\n" + c.push(data) + c.push(SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, []) + self.assertEqual(c.buffer, data) + + def test_simple_producer(self): + s, event = start_echo_server() + c = echo_client(b'\n', s.port) + data = b"hello world\nI'm not dead yet!\n" + p = asynchat.simple_producer(data+SERVER_QUIT, buffer_size=8) + c.push_with_producer(p) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) + + def test_string_producer(self): + s, event = start_echo_server() + c = echo_client(b'\n', s.port) + data = b"hello world\nI'm not dead yet!\n" + c.push_with_producer(data+SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, [b"hello world", b"I'm not dead yet!"]) + + def test_empty_line(self): + # checks that empty lines are handled correctly + s, event = start_echo_server() + c = echo_client(b'\n', s.port) + c.push(b"hello world\n\nI'm not dead yet!\n") + c.push(SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + + self.assertEqual(c.contents, + [b"hello world", b"", b"I'm not dead yet!"]) + + def test_close_when_done(self): + s, event = start_echo_server() + s.start_resend_event = threading.Event() + c = echo_client(b'\n', s.port) + c.push(b"hello world\nI'm not dead yet!\n") + c.push(SERVER_QUIT) + c.close_when_done() + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + + # Only allow the server to start echoing data back to the client after + # the client has closed its connection. This prevents a race condition + # where the server echoes all of its data before we can check that it + # got any down below. + s.start_resend_event.set() + threading_helper.join_thread(s) + + self.assertEqual(c.contents, []) + # the server might have been able to send a byte or two back, but this + # at least checks that it received something and didn't just fail + # (which could still result in the client not having received anything) + self.assertGreater(len(s.buffer), 0) + + def test_push(self): + # Issue #12523: push() should raise a TypeError if it doesn't get + # a bytes string + s, event = start_echo_server() + c = echo_client(b'\n', s.port) + data = b'bytes\n' + c.push(data) + c.push(bytearray(data)) + c.push(memoryview(data)) + self.assertRaises(TypeError, c.push, 10) + self.assertRaises(TypeError, c.push, 'unicode') + c.push(SERVER_QUIT) + asyncore.loop(use_poll=self.usepoll, count=300, timeout=.01) + threading_helper.join_thread(s) + self.assertEqual(c.contents, [b'bytes', b'bytes', b'bytes']) + + +class TestAsynchat_WithPoll(TestAsynchat): + usepoll = True + + +class TestAsynchatMocked(unittest.TestCase): + def test_blockingioerror(self): + # Issue #16133: handle_read() must ignore BlockingIOError + sock = unittest.mock.Mock() + sock.recv.side_effect = BlockingIOError(errno.EAGAIN) + + dispatcher = asynchat.async_chat() + dispatcher.set_socket(sock) + self.addCleanup(dispatcher.del_channel) + + with unittest.mock.patch.object(dispatcher, 'handle_error') as error: + dispatcher.handle_read() + self.assertFalse(error.called) + + +class TestHelperFunctions(unittest.TestCase): + def test_find_prefix_at_end(self): + self.assertEqual(asynchat.find_prefix_at_end("qwerty\r", "\r\n"), 1) + self.assertEqual(asynchat.find_prefix_at_end("qwertydkjf", "\r\n"), 0) + + +class TestNotConnected(unittest.TestCase): + def test_disallow_negative_terminator(self): + # Issue #11259 + client = asynchat.async_chat() + self.assertRaises(ValueError, client.set_terminator, -1) + + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py new file mode 100644 index 0000000..ecd1e12 --- /dev/null +++ b/Lib/test/test_asyncore.py @@ -0,0 +1,841 @@ +import unittest +import select +import os +import socket +import sys +import time +import errno +import struct +import threading + +from test import support +from test.support import os_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import warnings_helper +from io import BytesIO + +if support.PGO: + raise unittest.SkipTest("test is not helpful for PGO") + +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asyncore + + +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + +class dummysocket: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + def fileno(self): + return 42 + +class dummychannel: + def __init__(self): + self.socket = dummysocket() + + def close(self): + self.socket.close() + +class exitingdummy: + def __init__(self): + pass + + def handle_read_event(self): + raise asyncore.ExitNow() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + +class crashingdummy: + def __init__(self): + self.error_handled = False + + def handle_read_event(self): + raise Exception() + + handle_write_event = handle_read_event + handle_close = handle_read_event + handle_expt_event = handle_read_event + + def handle_error(self): + self.error_handled = True + +# used when testing senders; just collects what it gets until newline is sent +def capture_server(evt, buf, serv): + try: + serv.listen() + conn, addr = serv.accept() + except TimeoutError: + pass + else: + n = 200 + start = time.monotonic() + while n > 0 and time.monotonic() - start < 3.0: + r, w, e = select.select([conn], [], [], 0.1) + if r: + n -= 1 + data = conn.recv(10) + # keep everything except for the newline terminator + buf.write(data.replace(b'\n', b'')) + if b'\n' in data: + break + time.sleep(0.01) + + conn.close() + finally: + serv.close() + evt.set() + +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + os_helper.unlink(addr) + socket_helper.bind_unix_socket(sock, addr) + else: + sock.bind(addr) + + +class HelperFunctionTests(unittest.TestCase): + def test_readwriteexc(self): + # Check exception handling behavior of read, write and _exception + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore read/write/_exception calls + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.read, tr1) + self.assertRaises(asyncore.ExitNow, asyncore.write, tr1) + self.assertRaises(asyncore.ExitNow, asyncore._exception, tr1) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + asyncore.read(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore.write(tr2) + self.assertEqual(tr2.error_handled, True) + + tr2 = crashingdummy() + asyncore._exception(tr2) + self.assertEqual(tr2.error_handled, True) + + # asyncore.readwrite uses constants in the select module that + # are not present in Windows systems (see this thread: + # http://mail.python.org/pipermail/python-list/2001-October/109973.html) + # These constants should be present as long as poll is available + + @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') + def test_readwrite(self): + # Check that correct methods are called by readwrite() + + attributes = ('read', 'expt', 'write', 'closed', 'error_handled') + + expected = ( + (select.POLLIN, 'read'), + (select.POLLPRI, 'expt'), + (select.POLLOUT, 'write'), + (select.POLLERR, 'closed'), + (select.POLLHUP, 'closed'), + (select.POLLNVAL, 'closed'), + ) + + class testobj: + def __init__(self): + self.read = False + self.write = False + self.closed = False + self.expt = False + self.error_handled = False + + def handle_read_event(self): + self.read = True + + def handle_write_event(self): + self.write = True + + def handle_close(self): + self.closed = True + + def handle_expt_event(self): + self.expt = True + + def handle_error(self): + self.error_handled = True + + for flag, expectedattr in expected: + tobj = testobj() + self.assertEqual(getattr(tobj, expectedattr), False) + asyncore.readwrite(tobj, flag) + + # Only the attribute modified by the routine we expect to be + # called should be True. + for attr in attributes: + self.assertEqual(getattr(tobj, attr), attr==expectedattr) + + # check that ExitNow exceptions in the object handler method + # bubbles all the way up through asyncore readwrite call + tr1 = exitingdummy() + self.assertRaises(asyncore.ExitNow, asyncore.readwrite, tr1, flag) + + # check that an exception other than ExitNow in the object handler + # method causes the handle_error method to get called + tr2 = crashingdummy() + self.assertEqual(tr2.error_handled, False) + asyncore.readwrite(tr2, flag) + self.assertEqual(tr2.error_handled, True) + + def test_closeall(self): + self.closeall_check(False) + + def test_closeall_default(self): + self.closeall_check(True) + + def closeall_check(self, usedefault): + # Check that close_all() closes everything in a given map + + l = [] + testmap = {} + for i in range(10): + c = dummychannel() + l.append(c) + self.assertEqual(c.socket.closed, False) + testmap[i] = c + + if usedefault: + socketmap = asyncore.socket_map + try: + asyncore.socket_map = testmap + asyncore.close_all() + finally: + testmap, asyncore.socket_map = asyncore.socket_map, socketmap + else: + asyncore.close_all(testmap) + + self.assertEqual(len(testmap), 0) + + for c in l: + self.assertEqual(c.socket.closed, True) + + def test_compact_traceback(self): + try: + raise Exception("I don't like spam!") + except: + real_t, real_v, real_tb = sys.exc_info() + r = asyncore.compact_traceback() + else: + self.fail("Expected exception") + + (f, function, line), t, v, info = r + self.assertEqual(os.path.split(f)[-1], 'test_asyncore.py') + self.assertEqual(function, 'test_compact_traceback') + self.assertEqual(t, real_t) + self.assertEqual(v, real_v) + self.assertEqual(info, '[%s|%s|%s]' % (f, function, line)) + + +class DispatcherTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + def test_basic(self): + d = asyncore.dispatcher() + self.assertEqual(d.readable(), True) + self.assertEqual(d.writable(), True) + + def test_repr(self): + d = asyncore.dispatcher() + self.assertEqual(repr(d), '' % id(d)) + + def test_log(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log() (to stderr) + l1 = "Lovely spam! Wonderful spam!" + l2 = "I don't like spam!" + with support.captured_stderr() as stderr: + d.log(l1) + d.log(l2) + + lines = stderr.getvalue().splitlines() + self.assertEqual(lines, ['log: %s' % l1, 'log: %s' % l2]) + + def test_log_info(self): + d = asyncore.dispatcher() + + # capture output of dispatcher.log_info() (to stdout via print) + l1 = "Have you got anything without spam?" + l2 = "Why can't she have egg bacon spam and sausage?" + l3 = "THAT'S got spam in it!" + with support.captured_stdout() as stdout: + d.log_info(l1, 'EGGS') + d.log_info(l2) + d.log_info(l3, 'SPAM') + + lines = stdout.getvalue().splitlines() + expected = ['EGGS: %s' % l1, 'info: %s' % l2, 'SPAM: %s' % l3] + self.assertEqual(lines, expected) + + def test_unhandled(self): + d = asyncore.dispatcher() + d.ignore_log_types = () + + # capture output of dispatcher.log_info() (to stdout via print) + with support.captured_stdout() as stdout: + d.handle_expt() + d.handle_read() + d.handle_write() + d.handle_connect() + + lines = stdout.getvalue().splitlines() + expected = ['warning: unhandled incoming priority event', + 'warning: unhandled read event', + 'warning: unhandled write event', + 'warning: unhandled connect event'] + self.assertEqual(lines, expected) + + def test_strerror(self): + # refers to bug #8573 + err = asyncore._strerror(errno.EPERM) + if hasattr(os, 'strerror'): + self.assertEqual(err, os.strerror(errno.EPERM)) + err = asyncore._strerror(-1) + self.assertTrue(err != "") + + +class dispatcherwithsend_noread(asyncore.dispatcher_with_send): + def readable(self): + return False + + def handle_connect(self): + pass + + +class DispatcherWithSendTests(unittest.TestCase): + def setUp(self): + pass + + def tearDown(self): + asyncore.close_all() + + @threading_helper.reap_threads + def test_send(self): + evt = threading.Event() + sock = socket.socket() + sock.settimeout(3) + port = socket_helper.bind_port(sock) + + cap = BytesIO() + args = (evt, cap, sock) + t = threading.Thread(target=capture_server, args=args) + t.start() + try: + # wait a little longer for the server to initialize (it sometimes + # refuses connections on slow machines without this wait) + time.sleep(0.2) + + data = b"Suppose there isn't a 16-ton weight?" + d = dispatcherwithsend_noread() + d.create_socket() + d.connect((socket_helper.HOST, port)) + + # give time for socket to connect + time.sleep(0.1) + + d.send(data) + d.send(data) + d.send(b'\n') + + n = 1000 + while d.out_buffer and n > 0: + asyncore.poll() + n -= 1 + + evt.wait() + + self.assertEqual(cap.getvalue(), data*2) + finally: + threading_helper.join_thread(t) + + +@unittest.skipUnless(hasattr(asyncore, 'file_wrapper'), + 'asyncore.file_wrapper required') +class FileWrapperTest(unittest.TestCase): + def setUp(self): + self.d = b"It's not dead, it's sleeping!" + with open(os_helper.TESTFN, 'wb') as file: + file.write(self.d) + + def tearDown(self): + os_helper.unlink(os_helper.TESTFN) + + def test_recv(self): + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + w = asyncore.file_wrapper(fd) + os.close(fd) + + self.assertNotEqual(w.fd, fd) + self.assertNotEqual(w.fileno(), fd) + self.assertEqual(w.recv(13), b"It's not dead") + self.assertEqual(w.read(6), b", it's") + w.close() + self.assertRaises(OSError, w.read, 1) + + def test_send(self): + d1 = b"Come again?" + d2 = b"I want to buy some cheese." + fd = os.open(os_helper.TESTFN, os.O_WRONLY | os.O_APPEND) + w = asyncore.file_wrapper(fd) + os.close(fd) + + w.write(d1) + w.send(d2) + w.close() + with open(os_helper.TESTFN, 'rb') as file: + self.assertEqual(file.read(), self.d + d1 + d2) + + @unittest.skipUnless(hasattr(asyncore, 'file_dispatcher'), + 'asyncore.file_dispatcher required') + def test_dispatcher(self): + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + data = [] + class FileDispatcher(asyncore.file_dispatcher): + def handle_read(self): + data.append(self.recv(29)) + s = FileDispatcher(fd) + os.close(fd) + asyncore.loop(timeout=0.01, use_poll=True, count=2) + self.assertEqual(b"".join(data), self.d) + + def test_resource_warning(self): + # Issue #11453 + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + + os.close(fd) + with warnings_helper.check_warnings(('', ResourceWarning)): + f = None + support.gc_collect() + + def test_close_twice(self): + fd = os.open(os_helper.TESTFN, os.O_RDONLY) + f = asyncore.file_wrapper(fd) + os.close(fd) + + os.close(f.fd) # file_wrapper dupped fd + with self.assertRaises(OSError): + f.close() + + self.assertEqual(f.fd, -1) + # calling close twice should not fail + f.close() + + +class BaseTestHandler(asyncore.dispatcher): + + def __init__(self, sock=None): + asyncore.dispatcher.__init__(self, sock) + self.flag = False + + def handle_accept(self): + raise Exception("handle_accept not supposed to be called") + + def handle_accepted(self): + raise Exception("handle_accepted not supposed to be called") + + def handle_connect(self): + raise Exception("handle_connect not supposed to be called") + + def handle_expt(self): + raise Exception("handle_expt not supposed to be called") + + def handle_close(self): + raise Exception("handle_close not supposed to be called") + + def handle_error(self): + raise + + +class BaseServer(asyncore.dispatcher): + """A server which listens on an address and dispatches the + connection to a handler. + """ + + def __init__(self, family, addr, handler=BaseTestHandler): + asyncore.dispatcher.__init__(self) + self.create_socket(family) + self.set_reuse_addr() + bind_af_aware(self.socket, addr) + self.listen(5) + self.handler = handler + + @property + def address(self): + return self.socket.getsockname() + + def handle_accepted(self, sock, addr): + self.handler(sock) + + def handle_error(self): + raise + + +class BaseClient(BaseTestHandler): + + def __init__(self, family, address): + BaseTestHandler.__init__(self) + self.create_socket(family) + self.connect(address) + + def handle_connect(self): + pass + + +class BaseTestAPI: + + def tearDown(self): + asyncore.close_all(ignore_all=True) + + def loop_waiting_for_flag(self, instance, timeout=5): + timeout = float(timeout) / 100 + count = 100 + while asyncore.socket_map and count > 0: + asyncore.loop(timeout=0.01, count=1, use_poll=self.use_poll) + if instance.flag: + return + count -= 1 + time.sleep(timeout) + self.fail("flag not set") + + def test_handle_connect(self): + # make sure handle_connect is called on connect() + + class TestClient(BaseClient): + def handle_connect(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_accept(self): + # make sure handle_accept() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + def test_handle_accepted(self): + # make sure handle_accepted() is called when a client connects + + class TestListener(BaseTestHandler): + + def __init__(self, family, addr): + BaseTestHandler.__init__(self) + self.create_socket(family) + bind_af_aware(self.socket, addr) + self.listen(5) + self.address = self.socket.getsockname() + + def handle_accept(self): + asyncore.dispatcher.handle_accept(self) + + def handle_accepted(self, sock, addr): + sock.close() + self.flag = True + + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) + self.loop_waiting_for_flag(server) + + + def test_handle_read(self): + # make sure handle_read is called on data received + + class TestClient(BaseClient): + def handle_read(self): + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.send(b'x' * 1024) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_write(self): + # make sure handle_write is called + + class TestClient(BaseClient): + def handle_write(self): + self.flag = True + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close(self): + # make sure handle_close is called when the other end closes + # the connection + + class TestClient(BaseClient): + + def handle_read(self): + # in order to make handle_close be called we are supposed + # to make at least one recv() call + self.recv(1024) + + def handle_close(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.close() + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close_after_conn_broken(self): + # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and + # #11265). + + data = b'\0' * 128 + + class TestClient(BaseClient): + + def handle_write(self): + self.send(data) + + def handle_close(self): + self.flag = True + self.close() + + def handle_expt(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + + def handle_read(self): + self.recv(len(data)) + self.close() + + def writable(self): + return False + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + @unittest.skipIf(sys.platform.startswith("sunos"), + "OOB support is broken on Solaris") + def test_handle_expt(self): + # Make sure handle_expt is called on OOB data received. + # Note: this might fail on some platforms as OOB data is + # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + if sys.platform == "darwin" and self.use_poll: + self.skipTest("poll may fail on macOS; see issue #28087") + + class TestClient(BaseClient): + def handle_expt(self): + self.socket.recv(1024, socket.MSG_OOB) + self.flag = True + + class TestHandler(BaseTestHandler): + def __init__(self, conn): + BaseTestHandler.__init__(self, conn) + self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_error(self): + + class TestClient(BaseClient): + def handle_write(self): + 1.0 / 0 + def handle_error(self): + self.flag = True + try: + raise + except ZeroDivisionError: + pass + else: + raise Exception("exception not raised") + + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_connection_attributes(self): + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) + + # we start disconnected + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + # this can't be taken for granted across all platforms + #self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # execute some loops so that client connects to server + asyncore.loop(timeout=0.01, use_poll=self.use_poll, count=100) + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertTrue(client.connected) + self.assertFalse(client.accepting) + + # disconnect the client + client.close() + self.assertFalse(server.connected) + self.assertTrue(server.accepting) + self.assertFalse(client.connected) + self.assertFalse(client.accepting) + + # stop serving + server.close() + self.assertFalse(server.connected) + self.assertFalse(server.accepting) + + def test_create_socket(self): + s = asyncore.dispatcher() + s.create_socket(self.family) + self.assertEqual(s.socket.type, socket.SOCK_STREAM) + self.assertEqual(s.socket.family, self.family) + self.assertEqual(s.socket.gettimeout(), 0) + self.assertFalse(s.socket.get_inheritable()) + + def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + s1 = asyncore.dispatcher() + s1.create_socket(self.family) + s1.bind(self.addr) + s1.listen(5) + port = s1.socket.getsockname()[1] + + s2 = asyncore.dispatcher() + s2.create_socket(self.family) + # EADDRINUSE indicates the socket was correctly bound + self.assertRaises(OSError, s2.bind, (self.addr[0], port)) + + def test_set_reuse_addr(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + + with socket.socket(self.family) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except OSError: + unittest.skip("SO_REUSEADDR not supported on this platform") + else: + # if SO_REUSEADDR succeeded for sock we expect asyncore + # to do the same + s = asyncore.dispatcher(socket.socket(self.family)) + self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + s.socket.close() + s.create_socket(self.family) + s.set_reuse_addr() + self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR)) + + @threading_helper.reap_threads + def test_quick_connect(self): + # see: http://bugs.python.org/issue10340 + if self.family not in (socket.AF_INET, getattr(socket, "AF_INET6", object())): + self.skipTest("test specific to AF_INET and AF_INET6") + + server = BaseServer(self.family, self.addr) + # run the thread 500 ms: the socket should be connected in 200 ms + t = threading.Thread(target=lambda: asyncore.loop(timeout=0.1, + count=5)) + t.start() + try: + with socket.socket(self.family, socket.SOCK_STREAM) as s: + s.settimeout(.2) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, + struct.pack('ii', 1, 0)) + + try: + s.connect(server.address) + except OSError: + pass + finally: + threading_helper.join_thread(t) + +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (socket_helper.HOST, 0) + +@unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = (socket_helper.HOSTv6, 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = os_helper.TESTFN + + def tearDown(self): + os_helper.unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets, unittest.TestCase): + use_poll = True + +class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets, unittest.TestCase): + use_poll = True + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c9edfac..56e3d8a 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -18,13 +18,17 @@ except ImportError: from unittest import TestCase, skipUnless from test import support -from test.support import _asynchat as asynchat -from test.support import _asyncore as asyncore -from test.support import socket_helper from test.support import threading_helper +from test.support import socket_helper from test.support import warnings_helper from test.support.socket_helper import HOST, HOSTv6 +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asyncore + import asynchat + TIMEOUT = support.LOOPBACK_TIMEOUT DEFAULT_ENCODING = 'utf-8' diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index e709ea3..85b6e5f 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -42,8 +42,6 @@ import sys import tempfile from test.support.script_helper import assert_python_ok, assert_python_failure from test import support -from test.support import _asyncore as asyncore -from test.support import _smtpd as smtpd from test.support import os_helper from test.support import socket_helper from test.support import threading_helper @@ -61,6 +59,11 @@ from urllib.parse import urlparse, parse_qs from socketserver import (ThreadingUDPServer, DatagramRequestHandler, ThreadingTCPServer, StreamRequestHandler) +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asyncore + import smtpd + try: import win32evtlog, win32evtlogutil, pywintypes except ImportError: diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 5e15340..8da0aa3 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -30,8 +30,6 @@ import unittest import uuid import warnings from test import support -from test.support import _asynchat as asynchat -from test.support import _asyncore as asyncore from test.support import import_helper from test.support import os_helper from test.support import socket_helper @@ -39,6 +37,11 @@ from test.support import threading_helper from test.support import warnings_helper from platform import win32_is_iot +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asynchat + import asyncore + try: import resource except ImportError: diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 23c6be3..44cf523 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -12,12 +12,16 @@ import threading import unittest from unittest import TestCase, skipUnless from test import support as test_support -from test.support import _asynchat as asynchat -from test.support import _asyncore as asyncore from test.support import hashlib_helper from test.support import socket_helper from test.support import threading_helper +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asynchat + import asyncore + HOST = socket_helper.HOST PORT = 0 diff --git a/Lib/test/test_smtpd.py b/Lib/test/test_smtpd.py new file mode 100644 index 0000000..d2e150d --- /dev/null +++ b/Lib/test/test_smtpd.py @@ -0,0 +1,1018 @@ +import unittest +import textwrap +from test import support, mock_socket +from test.support import socket_helper +from test.support import warnings_helper +import socket +import io + +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import smtpd + import asyncore + + +class DummyServer(smtpd.SMTPServer): + def __init__(self, *args, **kwargs): + smtpd.SMTPServer.__init__(self, *args, **kwargs) + self.messages = [] + if self._decode_data: + self.return_status = 'return status' + else: + self.return_status = b'return status' + + def process_message(self, peer, mailfrom, rcpttos, data, **kw): + self.messages.append((peer, mailfrom, rcpttos, data)) + if data == self.return_status: + return '250 Okish' + if 'mail_options' in kw and 'SMTPUTF8' in kw['mail_options']: + return '250 SMTPUTF8 message okish' + + +class DummyDispatcherBroken(Exception): + pass + + +class BrokenDummyServer(DummyServer): + def listen(self, num): + raise DummyDispatcherBroken() + + +class SMTPDServerTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def test_process_message_unimplemented(self): + server = smtpd.SMTPServer((socket_helper.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + + write_line(b'HELO example') + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + self.assertRaises(NotImplementedError, write_line, b'spam\r\n.\r\n') + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, + smtpd.SMTPServer, + (socket_helper.HOST, 0), + ('b', 0), + enable_SMTPUTF8=True, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class DebuggingServerTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def send_data(self, channel, data, enable_SMTPUTF8=False): + def write_line(line): + channel.socket.queue_recv(line) + channel.handle_read() + write_line(b'EHLO example') + if enable_SMTPUTF8: + write_line(b'MAIL From:eggs@example BODY=8BITMIME SMTPUTF8') + else: + write_line(b'MAIL From:eggs@example') + write_line(b'RCPT To:spam@example') + write_line(b'DATA') + write_line(data) + write_line(b'.') + + def test_process_message_with_decode_data_true(self): + server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nhello\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + From: test + X-Peer: peer-address + + hello + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_decode_data_false(self): + server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n') + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def test_process_SMTPUTF8_message_with_enable_SMTPUTF8_true(self): + server = smtpd.DebuggingServer((socket_helper.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + with support.captured_stdout() as s: + self.send_data(channel, b'From: test\n\nh\xc3\xa9llo\xff\n', + enable_SMTPUTF8=True) + stdout = s.getvalue() + self.assertEqual(stdout, textwrap.dedent("""\ + ---------- MESSAGE FOLLOWS ---------- + mail options: ['BODY=8BITMIME', 'SMTPUTF8'] + b'From: test' + b'X-Peer: peer-address' + b'' + b'h\\xc3\\xa9llo\\xff' + ------------ END MESSAGE ------------ + """)) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + +class TestFamilyDetection(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") + def test_socket_uses_IPv6(self): + server = smtpd.SMTPServer((socket_helper.HOSTv6, 0), (socket_helper.HOSTv4, 0)) + self.assertEqual(server.socket.family, socket.AF_INET6) + + def test_socket_uses_IPv4(self): + server = smtpd.SMTPServer((socket_helper.HOSTv4, 0), (socket_helper.HOSTv6, 0)) + self.assertEqual(server.socket.family, socket.AF_INET) + + +class TestRcptOptionParsing(unittest.TestCase): + error_response = (b'555 RCPT TO parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_params_rejected(self): + server = DummyServer((socket_helper.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: foo=bar') + self.assertEqual(channel.socket.last, self.error_response) + + def test_nothing_accepted(self): + server = DummyServer((socket_helper.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + self.write_line(channel, b'MAIL from: size=20') + self.write_line(channel, b'RCPT to: ') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class TestMailOptionParsing(unittest.TestCase): + error_response = (b'555 MAIL FROM parameters not recognized or not ' + b'implemented\r\n') + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, channel, line): + channel.socket.queue_recv(line) + channel.handle_read() + + def test_with_decode_data_true(self): + server = DummyServer((socket_helper.HOST, 0), ('b', 0), decode_data=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, decode_data=True) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + b'MAIL from: size=20 BODY=UNKNOWN', + b'MAIL from: size=20 body=8bitmime', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line(channel, b'MAIL from: size=20') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_decode_data_false(self): + server = DummyServer((socket_helper.HOST, 0), ('b', 0)) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr) + self.write_line(channel, b'EHLO example') + for line in [ + b'MAIL from: size=20 SMTPUTF8', + b'MAIL from: size=20 SMTPUTF8 BODY=8BITMIME', + ]: + self.write_line(channel, line) + self.assertEqual(channel.socket.last, self.error_response) + self.write_line( + channel, + b'MAIL from: size=20 SMTPUTF8 BODY=UNKNOWN') + self.assertEqual( + channel.socket.last, + b'501 Error: BODY can only be one of 7BIT, 8BITMIME\r\n') + self.write_line( + channel, b'MAIL from: size=20 body=8bitmime') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + def test_with_enable_smtputf8_true(self): + server = DummyServer((socket_helper.HOST, 0), ('b', 0), enable_SMTPUTF8=True) + conn, addr = server.accept() + channel = smtpd.SMTPChannel(server, conn, addr, enable_SMTPUTF8=True) + self.write_line(channel, b'EHLO example') + self.write_line( + channel, + b'MAIL from: size=20 body=8bitmime smtputf8') + self.assertEqual(channel.socket.last, b'250 OK\r\n') + + +class SMTPDChannelTest(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_broken_connect(self): + self.assertRaises( + DummyDispatcherBroken, BrokenDummyServer, + (socket_helper.HOST, 0), ('b', 0), decode_data=True) + + def test_decode_data_and_enable_SMTPUTF8_raises(self): + self.assertRaises( + ValueError, smtpd.SMTPChannel, + self.server, self.channel.conn, self.channel.addr, + enable_SMTPUTF8=True, decode_data=True) + + def test_server_accept(self): + self.server.handle_accept() + + def test_missing_data(self): + self.write_line(b'') + self.assertEqual(self.channel.socket.last, + b'500 Error: bad syntax\r\n') + + def test_EHLO(self): + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, b'250 HELP\r\n') + + def test_EHLO_bad_syntax(self): + self.write_line(b'EHLO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: EHLO hostname\r\n') + + def test_EHLO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_EHLO_HELO_duplicate(self): + self.write_line(b'EHLO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO(self): + name = smtpd.socket.getfqdn() + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + '250 {}\r\n'.format(name).encode('ascii')) + + def test_HELO_EHLO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'EHLO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELP(self): + self.write_line(b'HELP') + self.assertEqual(self.channel.socket.last, + b'250 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELP_command(self): + self.write_line(b'HELP MAIL') + self.assertEqual(self.channel.socket.last, + b'250 Syntax: MAIL FROM:
\r\n') + + def test_HELP_command_unknown(self): + self.write_line(b'HELP SPAM') + self.assertEqual(self.channel.socket.last, + b'501 Supported commands: EHLO HELO MAIL RCPT ' + \ + b'DATA RSET NOOP QUIT VRFY\r\n') + + def test_HELO_bad_syntax(self): + self.write_line(b'HELO') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: HELO hostname\r\n') + + def test_HELO_duplicate(self): + self.write_line(b'HELO example') + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'503 Duplicate HELO/EHLO\r\n') + + def test_HELO_parameter_rejected_when_extensions_not_enabled(self): + self.extended_smtp = False + self.write_line(b'HELO example') + self.write_line(b'MAIL from: SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_allows_space_after_colon(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_extended_MAIL_allows_space_after_colon(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: size=20') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_NOOP(self): + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_HELO_NOOP(self): + self.write_line(b'HELO example') + self.write_line(b'NOOP') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_NOOP_bad_syntax(self): + self.write_line(b'NOOP hi') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: NOOP\r\n') + + def test_QUIT(self): + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_HELO_QUIT(self): + self.write_line(b'HELO example') + self.write_line(b'QUIT') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_QUIT_arg_ignored(self): + self.write_line(b'QUIT bye bye') + self.assertEqual(self.channel.socket.last, b'221 Bye\r\n') + + def test_bad_state(self): + self.channel.smtp_state = 'BAD STATE' + self.write_line(b'HELO example') + self.assertEqual(self.channel.socket.last, + b'451 Internal confusion\r\n') + + def test_command_too_long(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from: ' + + b'a' * self.channel.command_size_limit + + b'@example') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_limit_extended_with_SIZE(self): + self.write_line(b'EHLO example') + fill_len = self.channel.command_size_limit - len('MAIL from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 26) + + b'@example> SIZE=1234') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + + def test_MAIL_command_rejects_SMTPUTF8_by_default(self): + self.write_line(b'EHLO example') + self.write_line( + b'MAIL from: BODY=8BITMIME SMTPUTF8') + self.assertEqual(self.channel.socket.last[0:1], b'5') + + def test_data_longer_than_default_data_size_limit(self): + # Hack the default so we don't have to generate so much data. + self.channel.data_size_limit = 1048 + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'A' * self.channel.data_size_limit + + b'A\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + def test_MAIL_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=512') + self.assertEqual(self.channel.socket.last, + b'250 OK\r\n') + + def test_MAIL_invalid_size_parameter(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=invalid') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_RCPT_unknown_parameters(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 MAIL FROM parameters not recognized or not implemented\r\n') + + self.write_line(b'MAIL FROM:') + self.write_line(b'RCPT TO: ham=green') + self.assertEqual(self.channel.socket.last, + b'555 RCPT TO parameters not recognized or not implemented\r\n') + + def test_MAIL_size_parameter_larger_than_default_data_size_limit(self): + self.channel.data_size_limit = 1048 + self.write_line(b'EHLO example') + self.write_line(b'MAIL FROM: SIZE=2096') + self.assertEqual(self.channel.socket.last, + b'552 Error: message size exceeds fixed maximum message size\r\n') + + def test_need_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'RCPT to:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: need MAIL command\r\n') + + def test_MAIL_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
[SP ]\r\n') + + def test_MAIL_missing_address(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: MAIL FROM:
\r\n') + + def test_MAIL_chevrons(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_empty_chevrons(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from:<>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_MAIL_quoted_localpart(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: <"Fred Blogs"@example.com> SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_MAIL_quoted_localpart_with_size_no_angles(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL from: "Fred Blogs"@example.com SIZE=1000') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.channel.mailfrom, '"Fred Blogs"@example.com') + + def test_nested_MAIL(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL from:eggs@example') + self.write_line(b'MAIL from:spam@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: nested MAIL command\r\n') + + def test_VRFY(self): + self.write_line(b'VRFY eggs@example') + self.assertEqual(self.channel.socket.last, + b'252 Cannot VRFY user, but will accept message and attempt ' + \ + b'delivery\r\n') + + def test_VRFY_syntax(self): + self.write_line(b'VRFY') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: VRFY
\r\n') + + def test_EXPN_not_implemented(self): + self.write_line(b'EXPN') + self.assertEqual(self.channel.socket.last, + b'502 EXPN not implemented\r\n') + + def test_no_HELO_MAIL(self): + self.write_line(b'MAIL from:') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_need_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'503 Error: need RCPT command\r\n') + + def test_RCPT_syntax_HELO(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
\r\n') + + def test_RCPT_syntax_EHLO(self): + self.write_line(b'EHLO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'501 Syntax: RCPT TO:
[SP ]\r\n') + + def test_RCPT_lowercase_to_OK(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From: eggs@example') + self.write_line(b'RCPT to: ') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_no_HELO_RCPT(self): + self.write_line(b'RCPT to eggs@example') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_DATA_syntax(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, b'501 Syntax: DATA\r\n') + + def test_no_HELO_DATA(self): + self.write_line(b'DATA spam') + self.assertEqual(self.channel.socket.last, + b'503 Error: send HELO first\r\n') + + def test_data_transparency_section_4_5_2(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'..\r\n.\r\n') + self.assertEqual(self.channel.received_data, '.') + + def test_multiple_RCPT(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RCPT To:ham@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example','ham@example'], + 'data')]) + + def test_manual_status(self): + # checks that the Channel is able to return a custom status message + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'return status\r\n.') + self.assertEqual(self.channel.socket.last, b'250 Okish\r\n') + + def test_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'MAIL From:foo@example') + self.write_line(b'RCPT To:eggs@example') + self.write_line(b'DATA') + self.write_line(b'data\r\n.') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'foo@example', + ['eggs@example'], + 'data')]) + + def test_HELO_RSET(self): + self.write_line(b'HELO example') + self.write_line(b'RSET') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_RSET_syntax(self): + self.write_line(b'RSET hi') + self.assertEqual(self.channel.socket.last, b'501 Syntax: RSET\r\n') + + def test_unknown_command(self): + self.write_line(b'UNKNOWN_CMD') + self.assertEqual(self.channel.socket.last, + b'500 Error: command "UNKNOWN_CMD" not ' + \ + b'recognized\r\n') + + def test_attribute_deprecations(self): + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__server + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__server = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__line + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__line = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__state + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__state = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__greeting + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__greeting = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__mailfrom + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__mailfrom = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__rcpttos + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__rcpttos = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__data + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__data = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__fqdn + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__fqdn = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__peer + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__peer = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__conn + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__conn = 'spam' + with warnings_helper.check_warnings(('', DeprecationWarning)): + spam = self.channel._SMTPChannel__addr + with warnings_helper.check_warnings(('', DeprecationWarning)): + self.channel._SMTPChannel__addr = 'spam' + +@unittest.skipUnless(socket_helper.IPV6_ENABLED, "IPv6 not enabled") +class SMTPDChannelIPv6Test(SMTPDChannelTest): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOSTv6, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + +class SMTPDChannelWithDataSizeLimitTest(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set DATA size limit to 32 bytes for easy testing + self.channel = smtpd.SMTPChannel(self.server, conn, addr, 32, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_data_limit_dialog(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'data\r\nmore\r\n.') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.assertEqual(self.server.messages, + [(('peer-address', 'peer-port'), + 'eggs@example', + ['spam@example'], + 'data\nmore')]) + + def test_data_limit_dialog_too_much_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + self.write_line(b'RCPT To:spam@example') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last, + b'354 End data with .\r\n') + self.write_line(b'This message is longer than 32 bytes\r\n.') + self.assertEqual(self.channel.socket.last, + b'552 Error: Too much mail data\r\n') + + +class SMTPDChannelWithDecodeDataFalse(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOST, 0), ('b', 0)) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, b'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87\n' + b'and some plain ascii') + + +class SMTPDChannelWithDecodeDataTrue(unittest.TestCase): + + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), + decode_data=True) + conn, addr = self.server.accept() + # Set decode_data to True + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + decode_data=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_ascii_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'plain ascii text') + self.write_line(b'.') + self.assertEqual(self.channel.received_data, 'plain ascii text') + + def test_utf8_data(self): + self.write_line(b'HELO example') + self.write_line(b'MAIL From:eggs@example') + self.write_line(b'RCPT To:spam@example') + self.write_line(b'DATA') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'and some plain ascii') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + 'utf8 enriched text: żźć\nand some plain ascii') + + +class SMTPDChannelTestWithEnableSMTPUTF8True(unittest.TestCase): + def setUp(self): + smtpd.socket = asyncore.socket = mock_socket + self.old_debugstream = smtpd.DEBUGSTREAM + self.debug = smtpd.DEBUGSTREAM = io.StringIO() + self.server = DummyServer((socket_helper.HOST, 0), ('b', 0), + enable_SMTPUTF8=True) + conn, addr = self.server.accept() + self.channel = smtpd.SMTPChannel(self.server, conn, addr, + enable_SMTPUTF8=True) + + def tearDown(self): + asyncore.close_all() + asyncore.socket = smtpd.socket = socket + smtpd.DEBUGSTREAM = self.old_debugstream + + def write_line(self, line): + self.channel.socket.queue_recv(line) + self.channel.handle_read() + + def test_MAIL_command_accepts_SMTPUTF8_when_announced(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL from: BODY=8BITMIME SMTPUTF8'.encode( + 'utf-8') + ) + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_process_smtputf8_message(self): + self.write_line(b'EHLO example') + for mail_parameters in [b'', b'BODY=8BITMIME SMTPUTF8']: + self.write_line(b'MAIL from: ' + mail_parameters) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'c\r\n.') + if mail_parameters == b'': + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + else: + self.assertEqual(self.channel.socket.last, + b'250 SMTPUTF8 message okish\r\n') + + def test_utf8_data(self): + self.write_line(b'EHLO example') + self.write_line( + 'MAIL From: naïve@examplé BODY=8BITMIME SMTPUTF8'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line('RCPT To:späm@examplé'.encode('utf-8')) + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'DATA') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + self.write_line(b'.') + self.assertEqual( + self.channel.received_data, + b'utf8 enriched text: \xc5\xbc\xc5\xba\xc4\x87') + + def test_MAIL_command_limit_extended_with_SIZE_and_SMTPUTF8(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + self.write_line(b'MAIL from:<' + + b'a' * (fill_len + 1) + + b'@example>') + self.assertEqual(self.channel.socket.last, + b'500 Error: line too long\r\n') + self.write_line(b'MAIL from:<' + + b'a' * fill_len + + b'@example>') + self.assertEqual(self.channel.socket.last, b'250 OK\r\n') + + def test_multiple_emails_with_extended_command_length(self): + self.write_line(b'ehlo example') + fill_len = (512 + 26 + 10) - len('mail from:<@example>') + for char in [b'a', b'b', b'c']: + self.write_line(b'MAIL from:<' + char * fill_len + b'a@example>') + self.assertEqual(self.channel.socket.last[0:3], b'500') + self.write_line(b'MAIL from:<' + char * fill_len + b'@example>') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'rcpt to:') + self.assertEqual(self.channel.socket.last[0:3], b'250') + self.write_line(b'data') + self.assertEqual(self.channel.socket.last[0:3], b'354') + self.write_line(b'test\r\n.') + self.assertEqual(self.channel.socket.last[0:3], b'250') + + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + not_exported = { + "program", "Devnull", "DEBUGSTREAM", "NEWLINE", "COMMASPACE", + "DATA_SIZE_DEFAULT", "usage", "Options", "parseargs", + } + support.check__all__(self, smtpd, not_exported=not_exported) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index bdce3e3..9761a37 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -18,13 +18,17 @@ import threading import unittest from test import support, mock_socket -from test.support import _asyncore as asyncore -from test.support import _smtpd as smtpd from test.support import hashlib_helper from test.support import socket_helper from test.support import threading_helper from unittest.mock import Mock +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asyncore + import smtpd + HOST = socket_helper.HOST if sys.platform == 'darwin': diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 88eeb07..981e2fe 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -4,7 +4,6 @@ import sys import unittest import unittest.mock from test import support -from test.support import _asyncore as asyncore from test.support import import_helper from test.support import os_helper from test.support import socket_helper @@ -31,6 +30,10 @@ try: except ImportError: ctypes = None +import warnings +with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + import asyncore ssl = import_helper.import_module("ssl") import _ssl diff --git a/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst b/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst deleted file mode 100644 index 4924381..0000000 --- a/Misc/NEWS.d/next/Library/2021-11-11-12-59-10.bpo-28533.68mMZa.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove the ``asyncore`` and ``asynchat`` modules, deprecated in Python 3.6: -use the :mod:`asyncio` module instead. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst b/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst deleted file mode 100644 index 67fb368..0000000 --- a/Misc/NEWS.d/next/Library/2021-11-11-12-59-49.bpo-28533.LvIFCQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove the ``smtpd`` module, deprecated in Python 3.6: the `aiosmtpd -`__ module can be used instead, it is based -on asyncio. Patch by Victor Stinner. diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj index 7dd40ad..43c570f 100644 --- a/PCbuild/lib.pyproj +++ b/PCbuild/lib.pyproj @@ -24,6 +24,7 @@ + @@ -49,6 +50,7 @@ + @@ -720,6 +722,7 @@ + @@ -857,6 +860,7 @@ + @@ -878,6 +882,7 @@ + @@ -1258,6 +1263,7 @@ + diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 7d42108..754fa94 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -96,7 +96,9 @@ static const char* _Py_stdlib_module_names[] = { "argparse", "array", "ast", +"asynchat", "asyncio", +"asyncore", "atexit", "audioop", "base64", @@ -240,6 +242,7 @@ static const char* _Py_stdlib_module_names[] = { "shutil", "signal", "site", +"smtpd", "smtplib", "sndhdr", "socket", -- cgit v0.12