#14731: refactor email policy framework.

This patch primarily does two things: (1) it adds some internal-interface
methods to Policy that allow for Policy to control the parsing and folding of
headers in such a way that we can construct a backward compatibility policy
that is 100% compatible with the 3.2 API, while allowing a new policy to
implement the email6 API.  (2) it adds that backward compatibility policy and
refactors the test suite so that the only differences between the 3.2
test_email.py file and the 3.3 test_email.py file is some small changes in
test framework and the addition of tests for bugs fixed that apply to the 3.2
API.

There are some additional teaks, such as moving just the code needed for the
compatibility policy into _policybase, so that the library code can import
only _policybase.  That way the new code that will be added for email6
will only get imported when a non-compatibility policy is imported.

8 files changed, 649 insertions, 260 deletions

diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py
new file mode 100644
index 0000000..05736d0
--- /dev/null
+++ b/Lib/email/_policybase.py
@@ -0,0 +1,338 @@
+"""Policy framework for the email package.
+
+Allows fine grained feature control of how the package parses and emits data.
+"""
+
+import abc
+from email import header
+from email import charset as _charset
+from email.utils import _has_surrogates
+
+__all__ = [
+    'Policy',
+    'Compat32',
+    'compat32',
+    ]
+
+
+class _PolicyBase:
+
+    """Policy Object basic framework.
+
+    This class is useless unless subclassed.  A subclass should define
+    class attributes with defaults for any values that are to be
+    managed by the Policy object.  The constructor will then allow
+    non-default values to be set for these attributes at instance
+    creation time.  The instance will be callable, taking these same
+    attributes keyword arguments, and returning a new instance
+    identical to the called instance except for those values changed
+    by the keyword arguments.  Instances may be added, yielding new
+    instances with any non-default values from the right hand
+    operand overriding those in the left hand operand.  That is,
+
+        A + B == A(<non-default values of B>)
+
+    The repr of an instance can be used to reconstruct the object
+    if and only if the repr of the values can be used to reconstruct
+    those values.
+
+    """
+
+    def __init__(self, **kw):
+        """Create new Policy, possibly overriding some defaults.
+
+        See class docstring for a list of overridable attributes.
+
+        """
+        for name, value in kw.items():
+            if hasattr(self, name):
+                super(_PolicyBase,self).__setattr__(name, value)
+            else:
+                raise TypeError(
+                    "{!r} is an invalid keyword argument for {}".format(
+                        name, self.__class__.__name__))
+
+    def __repr__(self):
+        args = [ "{}={!r}".format(name, value)
+                 for name, value in self.__dict__.items() ]
+        return "{}({})".format(self.__class__.__name__, ', '.join(args))
+
+    def clone(self, **kw):
+        """Return a new instance with specified attributes changed.
+
+        The new instance has the same attribute values as the current object,
+        except for the changes passed in as keyword arguments.
+
+        """
+        for attr, value in self.__dict__.items():
+            if attr not in kw:
+                kw[attr] = value
+        return self.__class__(**kw)
+
+    def __setattr__(self, name, value):
+        if hasattr(self, name):
+            msg = "{!r} object attribute {!r} is read-only"
+        else:
+            msg = "{!r} object has no attribute {!r}"
+        raise AttributeError(msg.format(self.__class__.__name__, name))
+
+    def __add__(self, other):
+        """Non-default values from right operand override those from left.
+
+        The object returned is a new instance of the subclass.
+
+        """
+        return self.clone(**other.__dict__)
+
+
+# Conceptually this isn't a subclass of ABCMeta, but since we want Policy to
+# use ABCMeta as a metaclass *and* we want it to use this one as well, we have
+# to make this one a subclas of ABCMeta.
+class _DocstringExtenderMetaclass(abc.ABCMeta):
+
+    def __new__(meta, classname, bases, classdict):
+        if classdict.get('__doc__') and classdict['__doc__'].startswith('+'):
+            classdict['__doc__'] = meta._append_doc(bases[0].__doc__,
+                                                    classdict['__doc__'])
+        for name, attr in classdict.items():
+            if attr.__doc__ and attr.__doc__.startswith('+'):
+                for cls in (cls for base in bases for cls in base.mro()):
+                    doc = getattr(getattr(cls, name), '__doc__')
+                    if doc:
+                        attr.__doc__ = meta._append_doc(doc, attr.__doc__)
+                        break
+        return super().__new__(meta, classname, bases, classdict)
+
+    @staticmethod
+    def _append_doc(doc, added_doc):
+        added_doc = added_doc.split('\n', 1)[1]
+        return doc + '\n' + added_doc
+
+
+class Policy(_PolicyBase, metaclass=_DocstringExtenderMetaclass):
+
+    r"""Controls for how messages are interpreted and formatted.
+
+    Most of the classes and many of the methods in the email package accept
+    Policy objects as parameters.  A Policy object contains a set of values and
+    functions that control how input is interpreted and how output is rendered.
+    For example, the parameter 'raise_on_defect' controls whether or not an RFC
+    violation results in an error being raised or not, while 'max_line_length'
+    controls the maximum length of output lines when a Message is serialized.
+
+    Any valid attribute may be overridden when a Policy is created by passing
+    it as a keyword argument to the constructor.  Policy objects are immutable,
+    but a new Policy object can be created with only certain values changed by
+    calling the Policy instance with keyword arguments.  Policy objects can
+    also be added, producing a new Policy object in which the non-default
+    attributes set in the right hand operand overwrite those specified in the
+    left operand.
+
+    Settable attributes:
+
+    raise_on_defect     -- If true, then defects should be raised as errors.
+                           Default: False.
+
+    linesep             -- string containing the value to use as separation
+                           between output lines.  Default '\n'.
+
+    cte_type            -- Type of allowed content transfer encodings
+
+                           7bit  -- ASCII only
+                           8bit  -- Content-Transfer-Encoding: 8bit is allowed
+
+                           Default: 8bit.  Also controls the disposition of
+                           (RFC invalid) binary data in headers; see the
+                           documentation of the binary_fold method.
+
+    max_line_length     -- maximum length of lines, excluding 'linesep',
+                           during serialization.  None or 0 means no line
+                           wrapping is done.  Default is 78.
+
+    """
+
+    raise_on_defect = False
+    linesep = '\n'
+    cte_type = '8bit'
+    max_line_length = 78
+
+    def handle_defect(self, obj, defect):
+        """Based on policy, either raise defect or call register_defect.
+
+            handle_defect(obj, defect)
+
+        defect should be a Defect subclass, but in any case must be an
+        Exception subclass.  obj is the object on which the defect should be
+        registered if it is not raised.  If the raise_on_defect is True, the
+        defect is raised as an error, otherwise the object and the defect are
+        passed to register_defect.
+
+        This method is intended to be called by parsers that discover defects.
+        The email package parsers always call it with Defect instances.
+
+        """
+        if self.raise_on_defect:
+            raise defect
+        self.register_defect(obj, defect)
+
+    def register_defect(self, obj, defect):
+        """Record 'defect' on 'obj'.
+
+        Called by handle_defect if raise_on_defect is False.  This method is
+        part of the Policy API so that Policy subclasses can implement custom
+        defect handling.  The default implementation calls the append method of
+        the defects attribute of obj.  The objects used by the email package by
+        default that get passed to this method will always have a defects
+        attribute with an append method.
+
+        """
+        obj.defects.append(defect)
+
+    @abc.abstractmethod
+    def header_source_parse(self, sourcelines):
+        """Given a list of linesep terminated strings constituting the lines of
+        a single header, return the (name, value) tuple that should be stored
+        in the model.  The input lines should retain their terminating linesep
+        characters.  The lines passed in by the email package may contain
+        surrogateescaped binary data.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def header_store_parse(self, name, value):
+        """Given the header name and the value provided by the application
+        program, return the (name, value) that should be stored in the model.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def header_fetch_parse(self, name, value):
+        """Given the header name and the value from the model, return the value
+        to be returned to the application program that is requesting that
+        header.  The value passed in by the email package may contain
+        surrogateescaped binary data if the lines were parsed by a BytesParser.
+        The returned value should not contain any surrogateescaped data.
+
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def fold(self, name, value):
+        """Given the header name and the value from the model, return a string
+        containing linesep characters that implement the folding of the header
+        according to the policy controls.  The value passed in by the email
+        package may contain surrogateescaped binary data if the lines were
+        parsed by a BytesParser.  The returned value should not contain any
+        surrogateescaped data.
+
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def fold_binary(self, name, value):
+        """Given the header name and the value from the model, return binary
+        data containing linesep characters that implement the folding of the
+        header according to the policy controls.  The value passed in by the
+        email package may contain surrogateescaped binary data.
+
+        """
+        raise NotImplementedError
+
+
+class Compat32(Policy):
+
+    """+
+    This particular policy is the backward compatibility Policy.  It
+    replicates the behavior of the email package version 5.1.
+    """
+
+    def _sanitize_header(self, name, value):
+        # If the header value contains surrogates, return a Header using
+        # the unknown-8bit charset to encode the bytes as encoded words.
+        if not isinstance(value, str):
+            # Assume it is already a header object
+            return value
+        if _has_surrogates(value):
+            return header.Header(value, charset=_charset.UNKNOWN8BIT,
+                                 header_name=name)
+        else:
+            return value
+
+    def header_source_parse(self, sourcelines):
+        """+
+        The name is parsed as everything up to the ':' and returned unmodified.
+        The value is determined by stripping leading whitespace off the
+        remainder of the first line, joining all subsequent lines together, and
+        stripping any trailing carriage return or linefeed characters.
+
+        """
+        name, value = sourcelines[0].split(':', 1)
+        value = value.lstrip(' \t') + ''.join(sourcelines[1:])
+        return (name, value.rstrip('\r\n'))
+
+    def header_store_parse(self, name, value):
+        """+
+        The name and value are returned unmodified.
+        """
+        return (name, value)
+
+    def header_fetch_parse(self, name, value):
+        """+
+        If the value contains binary data, it is converted into a Header object
+        using the unknown-8bit charset.  Otherwise it is returned unmodified.
+        """
+        return self._sanitize_header(name, value)
+
+    def fold(self, name, value):
+        """+
+        Headers are folded using the Header folding algorithm, which preserves
+        existing line breaks in the value, and wraps each resulting line to the
+        max_line_length.  Non-ASCII binary data are CTE encoded using the
+        unknown-8bit charset.
+
+        """
+        return self._fold(name, value, sanitize=True)
+
+    def fold_binary(self, name, value):
+        """+
+        Headers are folded using the Header folding algorithm, which preserves
+        existing line breaks in the value, and wraps each resulting line to the
+        max_line_length.  If cte_type is 7bit, non-ascii binary data is CTE
+        encoded using the unknown-8bit charset.  Otherwise the original source
+        header is used, with its existing line breaks and/or binary data.
+
+        """
+        folded = self._fold(name, value, sanitize=self.cte_type=='7bit')
+        return folded.encode('ascii', 'surrogateescape')
+
+    def _fold(self, name, value, sanitize):
+        parts = []
+        parts.append('%s: ' % name)
+        if isinstance(value, str):
+            if _has_surrogates(value):
+                if sanitize:
+                    h = header.Header(value,
+                                      charset=_charset.UNKNOWN8BIT,
+                                      header_name=name)
+                else:
+                    # If we have raw 8bit data in a byte string, we have no idea
+                    # what the encoding is.  There is no safe way to split this
+                    # string.  If it's ascii-subset, then we could do a normal
+                    # ascii split, but if it's multibyte then we could break the
+                    # string.  There's no way to know so the least harm seems to
+                    # be to not split the string and risk it being too long.
+                    parts.append(value)
+                    h = None
+            else:
+                h = header.Header(value, header_name=name)
+        else:
+            # Assume it is a Header-like object.
+            h = value
+        if h is not None:
+            parts.append(h.encode(linesep=self.linesep,
+                                  maxlinelen=self.max_line_length))
+        parts.append(self.linesep)
+        return ''.join(parts)
+
+
+compat32 = Compat32()
diff --git a/Lib/email/architecture.rst b/Lib/email/architecture.rst
new file mode 100644
index 0000000..80d24fe
--- /dev/null
+++ b/Lib/email/architecture.rst
@@ -0,0 +1,216 @@
+:mod:`email` Package Architecture
+=================================
+
+Overview
+--------
+
+The email package consists of three major components:
+
+    Model
+        An object structure that represents an email message, and provides an
+        API for creating, querying, and modifying a message.
+
+    Parser
+        Takes a sequence of characters or bytes and produces a model of the
+        email message represented by those characters or bytes.
+
+    Generator
+        Takes a model and turns it into a sequence of characters or bytes.  The
+        sequence can either be intended for human consumption (a printable
+        unicode string) or bytes suitable for transmission over the wire.  In
+        the latter case all data is properly encoded using the content transfer
+        encodings specified by the relevant RFCs.
+
+Conceptually the package is organized around the model.  The model provides both
+"external" APIs intended for use by application programs using the library,
+and "internal" APIs intended for use by the Parser and Generator components.
+This division is intentionally a bit fuzy; the API described by this documentation
+is all a public, stable API.  This allows for an application with special needs
+to implement its own parser and/or generator.
+
+In addition to the three major functional components, there is a third key
+component to the architecture:
+
+    Policy
+        An object that specifies various behavioral settings and carries
+        implementations of various behavior-controlling methods.
+
+The Policy framework provides a simple and convenient way to control the
+behavior of the library, making it possible for the library to be used in a
+very flexible fashion while leveraging the common code required to parse,
+represent, and generate message-like objects.  For example, in addition to the
+default :rfc:`5322` email message policy, we also have a policy that manages
+HTTP headers in a fashion compliant with :rfc:`2616`.  Individual policy
+controls, such as the maximum line length produced by the generator, can also
+be controlled individually to meet specialized application requirements.
+
+
+The Model
+---------
+
+The message model is implemented by the :class:`~email.message.Message` class.
+The model divides a message into the two fundamental parts discussed by the
+RFC: the header section and the body.  The `Message` object acts as a
+pseudo-dictionary of named headers.  Its dictionary interface provides
+convenient access to individual headers by name.  However, all headers are kept
+internally in an ordered list, so that the information about the order of the
+headers in the original message is preserved.
+
+The `Message` object also has a `payload` that holds the body.  A `payload` can
+be one of two things: data, or a list of `Message` objects.  The latter is used
+to represent a multipart MIME message.  Lists can be nested arbitrarily deeply
+in order to represent the message, with all terminal leaves having non-list
+data payloads.
+
+
+Message Lifecycle
+-----------------
+
+The general lifecyle of a message is:
+
+    Creation
+        A `Message` object can be created by a Parser, or it can be
+        instantiated as an empty message by an application.
+
+    Manipulation
+        The application may examine one or more headers, and/or the
+        payload, and it may modify one or more headers and/or
+        the payload.  This may be done on the top level `Message`
+        object, or on any sub-object.
+
+    Finalization
+        The Model is converted into a unicode or binary stream,
+        or the model is discarded.
+
+
+
+Header Policy Control During Lifecycle
+--------------------------------------
+
+One of the major controls exerted by the Policy is the management of headers
+during the `Message` lifecycle.  Most applications don't need to be aware of
+this.
+
+A header enters the model in one of two ways: via a Parser, or by being set to
+a specific value by an application program after the Model already exists.
+Similarly, a header exits the model in one of two ways: by being serialized by
+a Generator, or by being retrieved from a Model by an application program.  The
+Policy object provides hooks for all four of these pathways.
+
+The model storage for headers is a list of (name, value) tuples.
+
+The Parser identifies headers during parsing, and passes them to the
+:meth:`~email.policy.Policy.header_source_parse` method of the Policy.  The
+result of that method is the (name, value) tuple to be stored in the model.
+
+When an application program supplies a header value (for example, through the
+`Message` object `__setitem__` interface), the name and the value are passed to
+the :meth:`~email.policy.Policy.header_store_parse` method of the Policy, which
+returns the (name, value) tuple to be stored in the model.
+
+When an application program retrieves a header (through any of the dict or list
+interfaces of `Message`), the name and value are passed to the
+:meth:`~email.policy.Policy.header_fetch_parse` method of the Policy to
+obtain the value returned to the application.
+
+When a Generator requests a header during serialization, the name and value are
+passed to the :meth:`~email.policy.Policy.fold` method of the Policy, which
+returns a string containing line breaks in the appropriate places.  The
+:meth:`~email.policy.Policy.cte_type` Policy control determines whether or
+not Content Transfer Encoding is performed on the data in the header.  There is
+also a :meth:`~email.policy.Policy.binary_fold` method for use by generators
+that produce binary output, which returns the folded header as binary data,
+possibly folded at different places than the corresponding string would be.
+
+
+Handling Binary Data
+--------------------
+
+In an ideal world all message data would conform to the RFCs, meaning that the
+parser could decode the message into the idealized unicode message that the
+sender originally wrote.  In the real world, the email package must also be
+able to deal with badly formatted messages, including messages containing
+non-ASCII characters that either have no indicated character set or are not
+valid characters in the indicated character set.
+
+Since email messages are *primarily* text data, and operations on message data
+are primarily text operations (except for binary payloads of course), the model
+stores all text data as unicode strings.  Un-decodable binary inside text
+data is handled by using the `surrogateescape` error handler of the ASCII
+codec.  As with the binary filenames the error handler was introduced to
+handle, this allows the email package to "carry" the binary data received
+during parsing along until the output stage, at which time it is regenerated
+in its original form.
+
+This carried binary data is almost entirely an implementation detail.  The one
+place where it is visible in the API is in the "internal" API.  A Parser must
+do the `surrogateescape` encoding of binary input data, and pass that data to
+the appropriate Policy method.  The "internal" interface used by the Generator
+to access header values preserves the `surrogateescaped` bytes.  All other
+interfaces convert the binary data either back into bytes or into a safe form
+(losing information in some cases).
+
+
+Backward Compatibility
+----------------------
+
+The :class:`~email.policy.Policy.Compat32` Policy provides backward
+compatibility with version 5.1 of the email package.  It does this via the
+following implementation of the four+1 Policy methods described above:
+
+header_source_parse
+    Splits the first line on the colon to obtain the name, discards any spaces
+    after the colon, and joins the remainder of the line with all of the
+    remaining lines, preserving the linesep characters to obtain the value.
+    Trailing carriage return and/or linefeed characters are stripped from the
+    resulting value string.
+
+header_store_parse
+    Returns the name and value exactly as received from the application.
+
+header_fetch_parse
+    If the value contains any `surrogateescaped` binary data, return the value
+    as a :class:`~email.header.Header` object, using the character set
+    `unknown-8bit`.  Otherwise just returns the value.
+
+fold
+    Uses :class:`~email.header.Header`'s folding to fold headers in the
+    same way the email5.1 generator did.
+
+binary_fold
+    Same as fold, but encodes to 'ascii'.
+
+
+New Algorithm
+-------------
+
+header_source_parse
+    Same as legacy behavior.
+
+header_store_parse
+    Same as legacy behavior.
+
+header_fetch_parse
+    If the value is already a header object, returns it.  Otherwise, parses the
+    value using the new parser, and returns the resulting object as the value.
+    `surrogateescaped` bytes get turned into unicode unknown character code
+    points.
+
+fold
+    Uses the new header folding algorithm, respecting the policy settings.
+    surrogateescaped bytes are encoded using the ``unknown-8bit`` charset for
+    ``cte_type=7bit`` or ``8bit``.  Returns a string.
+
+    At some point there will also be a ``cte_type=unicode``, and for that
+    policy fold will serialize the idealized unicode message with RFC-like
+    folding, converting any surrogateescaped bytes into the unicode
+    unknown character glyph.
+
+binary_fold
+    Uses the new header folding algorithm, respecting the policy settings.
+    surrogateescaped bytes are encoded using the `unknown-8bit` charset for
+    ``cte_type=7bit``, and get turned back into bytes for ``cte_type=8bit``.
+    Returns bytes.
+
+    At some point there will also be a ``cte_type=unicode``, and for that
+    policy binary_fold will serialize the message according to :rfc:``5335``.
diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py
index 533ebdf..0706cae 100644
--- a/Lib/email/feedparser.py
+++ b/Lib/email/feedparser.py
@@ -25,7 +25,7 @@ import re
 
 from email import errors
 from email import message
-from email import policy
+from email._policybase import compat32
 
 NLCRE = re.compile('\r\n|\r|\n')
 NLCRE_bol = re.compile('(\r\n|\r|\n)')
@@ -135,7 +135,7 @@ class BufferedSubFile(object):
 class FeedParser:
     """A feed-style parser of email."""
 
-    def __init__(self, _factory=message.Message, *, policy=policy.default):
+    def __init__(self, _factory=message.Message, *, policy=compat32):
         """_factory is called with no arguments to create a new message obj
 
         The policy keyword specifies a policy object that controls a number of
@@ -145,6 +145,12 @@ class FeedParser:
         """
         self._factory = _factory
         self.policy = policy
+        try:
+            _factory(policy=self.policy)
+            self._factory_kwds = lambda: {'policy': self.policy}
+        except TypeError:
+            # Assume this is an old-style factory
+            self._factory_kwds = lambda: {}
         self._input = BufferedSubFile()
         self._msgstack = []
         self._parse = self._parsegen().__next__
@@ -181,7 +187,7 @@ class FeedParser:
         return root
 
     def _new_message(self):
-        msg = self._factory()
+        msg = self._factory(**self._factory_kwds())
         if self._cur and self._cur.get_content_type() == 'multipart/digest':
             msg.set_default_type('message/rfc822')
         if self._msgstack:
@@ -458,9 +464,7 @@ class FeedParser:
                 lastvalue.append(line)
                 continue
             if lastheader:
-                # XXX reconsider the joining of folded lines
-                lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')
-                self._cur[lastheader] = lhdr
+                self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
                 lastheader, lastvalue = '', []
             # Check for envelope header, i.e. unix-from
             if line.startswith('From '):
@@ -487,16 +491,16 @@ class FeedParser:
             i = line.find(':')
             if i < 0:
                 defect = errors.MalformedHeaderDefect(line)
+                # XXX: fixme (defect not going through policy)
                 self._cur.defects.append(defect)
                 continue
             lastheader = line[:i]
-            lastvalue = [line[i+1:].lstrip()]
+            lastvalue = [line]
         # Done with all the lines, so handle the last header.
         if lastheader:
-            # XXX reconsider the joining of folded lines
-            self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')
+            self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
+
 
-
 class BytesFeedParser(FeedParser):
     """Like FeedParser, but feed accepts bytes."""
 
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index dcfea4c..bfa288b 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -13,9 +13,9 @@ import random
 import warnings
 
 from io import StringIO, BytesIO
-from email import policy
+from email._policybase import compat32
 from email.header import Header
-from email.message import _has_surrogates
+from email.utils import _has_surrogates
 import email.charset as _charset
 
 UNDERSCORE = '_'
@@ -36,7 +36,7 @@ class Generator:
     #
 
     def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *,
-                 policy=policy.default):
+                 policy=None):
         """Create the generator for message flattening.
 
         outfp is the output file-like object for writing the message to.  It
@@ -60,8 +60,7 @@ class Generator:
         """
         self._fp = outfp
         self._mangle_from_ = mangle_from_
-        self._maxheaderlen = (maxheaderlen if maxheaderlen is not None else
-                                 policy.max_line_length)
+        self.maxheaderlen = maxheaderlen
         self.policy = policy
 
     def write(self, s):
@@ -87,20 +86,33 @@ class Generator:
         # from the msg, and _encoded_XXX constants for operating on data that
         # has already been converted (to bytes in the BytesGenerator) and
         # inserted into a temporary buffer.
-        self._NL = linesep if linesep is not None else self.policy.linesep
+        policy = msg.policy if self.policy is None else self.policy
+        if linesep is not None:
+            policy = policy.clone(linesep=linesep)
+        if self.maxheaderlen is not None:
+            policy = policy.clone(max_line_length=self.maxheaderlen)
+        self._NL = policy.linesep
         self._encoded_NL = self._encode(self._NL)
         self._EMPTY = ''
         self._encoded_EMTPY = self._encode('')
-        if unixfrom:
-            ufrom = msg.get_unixfrom()
-            if not ufrom:
-                ufrom = 'From nobody ' + time.ctime(time.time())
-            self.write(ufrom + self._NL)
-        self._write(msg)
+        p = self.policy
+        try:
+            self.policy = policy
+            if unixfrom:
+                ufrom = msg.get_unixfrom()
+                if not ufrom:
+                    ufrom = 'From nobody ' + time.ctime(time.time())
+                self.write(ufrom + self._NL)
+            self._write(msg)
+        finally:
+            self.policy = p
 
     def clone(self, fp):
         """Clone this generator with the exact same options."""
-        return self.__class__(fp, self._mangle_from_, self._maxheaderlen)
+        return self.__class__(fp,
+                              self._mangle_from_,
+                              None, # Use policy setting, which we've adjusted
+                              policy=self.policy)
 
     #
     # Protected interface - undocumented ;/
@@ -175,16 +187,8 @@ class Generator:
     #
 
     def _write_headers(self, msg):
-        for h, v in msg.items():
-            self.write('%s: ' % h)
-            if isinstance(v, Header):
-                self.write(v.encode(
-                    maxlinelen=self._maxheaderlen, linesep=self._NL)+self._NL)
-            else:
-                # Header's got lots of smarts, so use it.
-                header = Header(v, maxlinelen=self._maxheaderlen,
-                                header_name=h)
-                self.write(header.encode(linesep=self._NL)+self._NL)
+        for h, v in msg.raw_items():
+            self.write(self.policy.fold(h, v))
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -265,12 +269,12 @@ class Generator:
         # The contents of signed parts has to stay unmodified in order to keep
         # the signature intact per RFC1847 2.1, so we disable header wrapping.
         # RDM: This isn't enough to completely preserve the part, but it helps.
-        old_maxheaderlen = self._maxheaderlen
+        p = self.policy
+        self.policy = p.clone(max_line_length=0)
         try:
-            self._maxheaderlen = 0
             self._handle_multipart(msg)
         finally:
-            self._maxheaderlen = old_maxheaderlen
+            self.policy = p
 
     def _handle_message_delivery_status(self, msg):
         # We can't just write the headers directly to self's file object
@@ -347,9 +351,9 @@ class BytesGenerator(Generator):
     Functionally identical to the base Generator except that the output is
     bytes and not string.  When surrogates were used in the input to encode
     bytes, these are decoded back to bytes for output.  If the policy has
-    must_be_7bit set true, then the message is transformed such that the
-    non-ASCII bytes are properly content transfer encoded, using the
-    charset unknown-8bit.
+    cte_type set to 7bit, then the message is transformed such that the
+    non-ASCII bytes are properly content transfer encoded, using the charset
+    unknown-8bit.
 
     The outfp object must accept bytes in its write method.
     """
@@ -370,27 +374,8 @@ class BytesGenerator(Generator):
     def _write_headers(self, msg):
         # This is almost the same as the string version, except for handling
         # strings with 8bit bytes.
-        for h, v in msg._headers:
-            self.write('%s: ' % h)
-            if isinstance(v, str):
-                if _has_surrogates(v):
-                    if not self.policy.must_be_7bit:
-                        # If we have raw 8bit data in a byte string, we have no idea
-                        # what the encoding is.  There is no safe way to split this
-                        # string.  If it's ascii-subset, then we could do a normal
-                        # ascii split, but if it's multibyte then we could break the
-                        # string.  There's no way to know so the least harm seems to
-                        # be to not split the string and risk it being too long.
-                        self.write(v+NL)
-                        continue
-                    h = Header(v, charset=_charset.UNKNOWN8BIT, header_name=h)
-                else:
-                    h = Header(v, header_name=h)
-            else:
-                # Assume it is a Header-like object.
-                h = v
-            self.write(h.encode(linesep=self._NL,
-                                maxlinelen=self._maxheaderlen)+self._NL)
+        for h, v in msg.raw_items():
+            self._fp.write(self.policy.fold_binary(h, v))
         # A blank line always separates headers from body
         self.write(self._NL)
 
@@ -399,7 +384,7 @@ class BytesGenerator(Generator):
         # just write it back out.
         if msg._payload is None:
             return
-        if _has_surrogates(msg._payload) and not self.policy.must_be_7bit:
+        if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit':
             self.write(msg._payload)
         else:
             super(BytesGenerator,self)._handle_text(msg)
diff --git a/Lib/email/message.py b/Lib/email/message.py
index 922617a..91976f1 100644
--- a/Lib/email/message.py
+++ b/Lib/email/message.py
@@ -10,13 +10,12 @@ import re
 import uu
 import base64
 import binascii
-import warnings
 from io import BytesIO, StringIO
 
 # Intrapackage imports
 from email import utils
 from email import errors
-from email import header
+from email._policybase import compat32
 from email import charset as _charset
 Charset = _charset.Charset
 
@@ -26,24 +25,6 @@ SEMISPACE = '; '
 # existence of which force quoting of the parameter value.
 tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]')
 
-# How to figure out if we are processing strings that come from a byte
-# source with undecodable characters.
-_has_surrogates = re.compile(
-    '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
-
-
-# Helper functions
-def _sanitize_header(name, value):
-    # If the header value contains surrogates, return a Header using
-    # the unknown-8bit charset to encode the bytes as encoded words.
-    if not isinstance(value, str):
-        # Assume it is already a header object
-        return value
-    if _has_surrogates(value):
-        return header.Header(value, charset=_charset.UNKNOWN8BIT,
-                             header_name=name)
-    else:
-        return value
 
 def _splitparam(param):
     # Split header parameters.  BAW: this may be too simple.  It isn't
@@ -136,7 +117,8 @@ class Message:
     you must use the explicit API to set or get all the headers.  Not all of
     the mapping methods are implemented.
     """
-    def __init__(self):
+    def __init__(self, policy=compat32):
+        self.policy = policy
         self._headers = []
         self._unixfrom = None
         self._payload = None
@@ -246,7 +228,7 @@ class Message:
         cte = str(self.get('content-transfer-encoding', '')).lower()
         # payload may be bytes here.
         if isinstance(payload, str):
-            if _has_surrogates(payload):
+            if utils._has_surrogates(payload):
                 bpayload = payload.encode('ascii', 'surrogateescape')
                 if not decode:
                     try:
@@ -362,7 +344,7 @@ class Message:
         Note: this does not overwrite an existing header with the same field
         name.  Use __delitem__() first to delete any existing headers.
         """
-        self._headers.append((name, val))
+        self._headers.append(self.policy.header_store_parse(name, val))
 
     def __delitem__(self, name):
         """Delete all occurrences of a header, if present.
@@ -401,7 +383,8 @@ class Message:
         Any fields deleted and re-inserted are always appended to the header
         list.
         """
-        return [_sanitize_header(k, v) for k, v in self._headers]
+        return [self.policy.header_fetch_parse(k, v)
+                for k, v in self._headers]
 
     def items(self):
         """Get all the message's header fields and values.
@@ -411,7 +394,8 @@ class Message:
         Any fields deleted and re-inserted are always appended to the header
         list.
         """
-        return [(k, _sanitize_header(k, v)) for k, v in self._headers]
+        return [(k, self.policy.header_fetch_parse(k, v))
+                for k, v in self._headers]
 
     def get(self, name, failobj=None):
         """Get a header value.
@@ -422,10 +406,29 @@ class Message:
         name = name.lower()
         for k, v in self._headers:
             if k.lower() == name:
-                return _sanitize_header(k, v)
+                return self.policy.header_fetch_parse(k, v)
         return failobj
 
     #
+    # "Internal" methods (public API, but only intended for use by a parser
+    # or generator, not normal application code.
+    #
+
+    def set_raw(self, name, value):
+        """Store name and value in the model without modification.
+
+        This is an "internal" API, intended only for use by a parser.
+        """
+        self._headers.append((name, value))
+
+    def raw_items(self):
+        """Return the (name, value) header pairs without modification.
+
+        This is an "internal" API, intended only for use by a generator.
+        """
+        return iter(self._headers.copy())
+
+    #
     # Additional useful stuff
     #
 
@@ -442,7 +445,7 @@ class Message:
         name = name.lower()
         for k, v in self._headers:
             if k.lower() == name:
-                values.append(_sanitize_header(k, v))
+                values.append(self.policy.header_fetch_parse(k, v))
         if not values:
             return failobj
         return values
@@ -475,7 +478,7 @@ class Message:
                 parts.append(_formatparam(k.replace('_', '-'), v))
         if _value is not None:
             parts.insert(0, _value)
-        self._headers.append((_name, SEMISPACE.join(parts)))
+        self[_name] = SEMISPACE.join(parts)
 
     def replace_header(self, _name, _value):
         """Replace a header.
@@ -487,7 +490,7 @@ class Message:
         _name = _name.lower()
         for i, (k, v) in zip(range(len(self._headers)), self._headers):
             if k.lower() == _name:
-                self._headers[i] = (k, _value)
+                self._headers[i] = self.policy.header_store_parse(k, _value)
                 break
         else:
             raise KeyError(_name)
@@ -805,7 +808,8 @@ class Message:
                         parts.append(k)
                     else:
                         parts.append('%s=%s' % (k, v))
-                newheaders.append((h, SEMISPACE.join(parts)))
+                val = SEMISPACE.join(parts)
+                newheaders.append(self.policy.header_store_parse(h, val))
 
             else:
                 newheaders.append((h, v))
diff --git a/Lib/email/parser.py b/Lib/email/parser.py
index 0f92160..1aab012 100644
--- a/Lib/email/parser.py
+++ b/Lib/email/parser.py
@@ -11,12 +11,12 @@ from io import StringIO, TextIOWrapper
 
 from email.feedparser import FeedParser
 from email.message import Message
-from email import policy
+from email._policybase import compat32
 
 
 
 class Parser:
-    def __init__(self, _class=Message, *, policy=policy.default):
+    def __init__(self, _class=Message, *, policy=compat32):
         """Parser of RFC 2822 and MIME email messages.
 
         Creates an in-memory object tree representing the email message, which
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
index 585a752..dae2dc7 100644
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -1,174 +1,12 @@
-"""Policy framework for the email package.
-
-Allows fine grained feature control of how the package parses and emits data.
+"""This will be the home for the policy that hooks in the new
+code that adds all the email6 features.
 """
 
-__all__ = [
-    'Policy',
-    'default',
-    'strict',
-    'SMTP',
-    'HTTP',
-    ]
-
-
-class _PolicyBase:
-
-    """Policy Object basic framework.
-
-    This class is useless unless subclassed.  A subclass should define
-    class attributes with defaults for any values that are to be
-    managed by the Policy object.  The constructor will then allow
-    non-default values to be set for these attributes at instance
-    creation time.  The instance will be callable, taking these same
-    attributes keyword arguments, and returning a new instance
-    identical to the called instance except for those values changed
-    by the keyword arguments.  Instances may be added, yielding new
-    instances with any non-default values from the right hand
-    operand overriding those in the left hand operand.  That is,
-
-        A + B == A(<non-default values of B>)
-
-    The repr of an instance can be used to reconstruct the object
-    if and only if the repr of the values can be used to reconstruct
-    those values.
-
-    """
-
-    def __init__(self, **kw):
-        """Create new Policy, possibly overriding some defaults.
-
-        See class docstring for a list of overridable attributes.
-
-        """
-        for name, value in kw.items():
-            if hasattr(self, name):
-                super(_PolicyBase,self).__setattr__(name, value)
-            else:
-                raise TypeError(
-                    "{!r} is an invalid keyword argument for {}".format(
-                        name, self.__class__.__name__))
-
-    def __repr__(self):
-        args = [ "{}={!r}".format(name, value)
-                 for name, value in self.__dict__.items() ]
-        return "{}({})".format(self.__class__.__name__, ', '.join(args))
-
-    def clone(self, **kw):
-        """Return a new instance with specified attributes changed.
-
-        The new instance has the same attribute values as the current object,
-        except for the changes passed in as keyword arguments.
-
-        """
-        for attr, value in self.__dict__.items():
-            if attr not in kw:
-                kw[attr] = value
-        return self.__class__(**kw)
-
-    def __setattr__(self, name, value):
-        if hasattr(self, name):
-            msg = "{!r} object attribute {!r} is read-only"
-        else:
-            msg = "{!r} object has no attribute {!r}"
-        raise AttributeError(msg.format(self.__class__.__name__, name))
-
-    def __add__(self, other):
-        """Non-default values from right operand override those from left.
-
-        The object returned is a new instance of the subclass.
-
-        """
-        return self.clone(**other.__dict__)
-
-
-class Policy(_PolicyBase):
-
-    """Controls for how messages are interpreted and formatted.
-
-    Most of the classes and many of the methods in the email package
-    accept Policy objects as parameters.  A Policy object contains a set
-    of values and functions that control how input is interpreted and how
-    output is rendered.  For example, the parameter 'raise_on_defect'
-    controls whether or not an RFC violation throws an error or not,
-    while 'max_line_length' controls the maximum length of output lines
-    when a Message is serialized.
-
-    Any valid attribute may be overridden when a Policy is created by
-    passing it as a keyword argument to the constructor.  Policy
-    objects are immutable, but a new Policy object can be created
-    with only certain values changed by calling the Policy instance
-    with keyword arguments.  Policy objects can also be added,
-    producing a new Policy object in which the non-default attributes
-    set in the right hand operand overwrite those specified in the
-    left operand.
-
-    Settable attributes:
-
-    raise_on_defect     -- If true, then defects should be raised
-                           as errors.  Default False.
-
-    linesep             -- string containing the value to use as
-                           separation between output lines.  Default '\n'.
-
-    must_be_7bit        -- output must contain only 7bit clean data.
-                           Default False.
-
-    max_line_length     -- maximum length of lines, excluding 'linesep',
-                           during serialization.  None means no line
-                           wrapping is done.  Default is 78.
-
-    Methods:
-
-    register_defect(obj, defect)
-        defect is a Defect instance.  The default implementation appends defect
-        to the objs 'defects' attribute.
-
-    handle_defect(obj, defect)
-        intended to be called by parser code that finds a defect.  If
-        raise_on_defect is True, defect is raised as an error, otherwise
-        register_defect is called.
-
-    """
-
-    raise_on_defect = False
-    linesep = '\n'
-    must_be_7bit = False
-    max_line_length = 78
-
-    def handle_defect(self, obj, defect):
-        """Based on policy, either raise defect or call register_defect.
-
-            handle_defect(obj, defect)
-
-        defect should be a Defect subclass, but in any case must be an
-        Exception subclass.  obj is the object on which the defect should be
-        registered if it is not raised.  If the raise_on_defect is True, the
-        defect is raised as an error, otherwise the object and the defect are
-        passed to register_defect.
-
-        This class is intended to be called by parsers that discover defects,
-        and will not be called from code using the library unless that code is
-        implementing an alternate parser.
-
-        """
-        if self.raise_on_defect:
-            raise defect
-        self.register_defect(obj, defect)
-
-    def register_defect(self, obj, defect):
-        """Record 'defect' on 'obj'.
-
-        Called by handle_defect if raise_on_defect is False.  This method is
-        part of the Policy API so that Policy subclasses can implement custom
-        defect handling.  The default implementation calls the append method
-        of the defects attribute of obj.
-
-        """
-        obj.defects.append(defect)
+from email._policybase import Policy, compat32, Compat32
 
+# XXX: temporarily derive everything from compat32.
 
-default = Policy()
+default = compat32
 strict = default.clone(raise_on_defect=True)
 SMTP = default.clone(linesep='\r\n')
 HTTP = default.clone(linesep='\r\n', max_line_length=None)
diff --git a/Lib/email/utils.py b/Lib/email/utils.py
index 138f05d..b82d5c5 100644
--- a/Lib/email/utils.py
+++ b/Lib/email/utils.py
@@ -57,6 +57,10 @@ TICK = "'"
 specialsre = re.compile(r'[][\\()<>@,:;".]')
 escapesre = re.compile(r'[\\"]')
 
+# How to figure out if we are processing strings that come from a byte
+# source with undecodable characters.
+_has_surrogates = re.compile(
+    '([^\ud800-\udbff]|\A)[\udc00-\udfff]([^\udc00-\udfff]|\Z)').search
 
 
 # Helpers