From 6074ee4ef2c451d5b8f92b53c22109163ed115c8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 23 Mar 2013 08:57:00 -0700 Subject: Update collections ABC table to match the __abstractmethods__ attribute for each container. --- Doc/library/collections.rst | 48 +++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 0fc7fac..50115ab 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -895,29 +895,35 @@ ABC Inherits from Abstract Methods Mixin :class:`Sized` ``__len__`` :class:`Callable` ``__call__`` -:class:`Sequence` :class:`Sized`, ``__getitem__`` ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Iterable`, ``index``, and ``count`` +:class:`Sequence` :class:`Sized`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, + :class:`Iterable`, ``__len__`` ``index``, and ``count`` :class:`Container` -:class:`MutableSequence` :class:`Sequence` ``__setitem__``, Inherited :class:`Sequence` methods and - ``__delitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``insert`` ``remove``, and ``__iadd__`` - -:class:`Set` :class:`Sized`, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, - :class:`Iterable`, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - :class:`Container` ``__sub__``, ``__xor__``, and ``isdisjoint`` - -:class:`MutableSet` :class:`Set` ``add``, Inherited :class:`Set` methods and - ``discard`` ``clear``, ``pop``, ``remove``, ``__ior__``, - ``__iand__``, ``__ixor__``, and ``__isub__`` - -:class:`Mapping` :class:`Sized`, ``__getitem__`` ``__contains__``, ``keys``, ``items``, ``values``, - :class:`Iterable`, ``get``, ``__eq__``, and ``__ne__`` - :class:`Container` - -:class:`MutableMapping` :class:`Mapping` ``__setitem__``, Inherited :class:`Mapping` methods and - ``__delitem__`` ``pop``, ``popitem``, ``clear``, ``update``, - and ``setdefault`` +:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and + ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, + ``__delitem__``, ``remove``, and ``__iadd__`` + ``__len__``, + ``insert`` + +:class:`Set` :class:`Sized`, ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, + :class:`Iterable`, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, + :class:`Container` ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` + +:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and + ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, + ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` + ``add``, + ``discard`` + +:class:`Mapping` :class:`Sized`, ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, + :class:`Iterable`, ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` + :class:`Container` ``__len__`` + +:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and + ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, + ``__delitem__``, and ``setdefault`` + ``__iter__``, + ``__len__`` :class:`MappingView` :class:`Sized` ``__len__`` -- cgit v0.12 From 92acb0a81c4905f827167eaac63c175030562f18 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Tue, 26 Mar 2013 17:53:05 +0100 Subject: Issue 17538: Document XML vulnerabilties --- Doc/library/markup.rst | 1 + Doc/library/pyexpat.rst | 8 +++ Doc/library/xml.dom.minidom.rst | 8 +++ Doc/library/xml.dom.pulldom.rst | 7 ++ Doc/library/xml.etree.elementtree.rst | 8 +++ Doc/library/xml.rst | 131 ++++++++++++++++++++++++++++++++++ Doc/library/xml.sax.rst | 8 +++ Doc/library/xmlrpclib.rst | 7 ++ Misc/NEWS | 5 ++ 9 files changed, 183 insertions(+) create mode 100644 Doc/library/xml.rst diff --git a/Doc/library/markup.rst b/Doc/library/markup.rst index 6782e39..0d05ef1 100644 --- a/Doc/library/markup.rst +++ b/Doc/library/markup.rst @@ -25,6 +25,7 @@ definition of the Python bindings for the DOM and SAX interfaces. htmlparser.rst sgmllib.rst htmllib.rst + xml.rst xml.etree.elementtree.rst xml.dom.rst xml.dom.minidom.rst diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index d09610c..c9136a8 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -14,6 +14,14 @@ directive. Since they are attributes which are set by client code, in-text references to these attributes should be marked using the :member: role. + +.. warning:: + + The :mod:`pyexpat` module is not secure against maliciously + constructed data. If you need to parse untrusted or unauthenticated data see + :ref:`xml-vulnerabilities`. + + .. versionadded:: 2.0 .. index:: single: Expat diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index bcc2ec6..70f0c03 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -20,6 +20,14 @@ to be simpler than the full DOM and also significantly smaller. Users who are not already proficient with the DOM should consider using the :mod:`xml.etree.ElementTree` module for their XML processing instead + +.. warning:: + + The :mod:`xml.dom.minidom` module is not secure against + maliciously constructed data. If you need to parse untrusted or + unauthenticated data see :ref:`xml-vulnerabilities`. + + DOM applications typically start by parsing some XML into a DOM. With :mod:`xml.dom.minidom`, this is done through the parse functions:: diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index bad0daa..9032706 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -16,6 +16,13 @@ Object Model representation of a document from SAX events. +.. warning:: + + The :mod:`xml.dom.pulldom` module is not secure against + maliciously constructed data. If you need to parse untrusted or + unauthenticated data see :ref:`xml-vulnerabilities`. + + .. class:: PullDOM([documentFactory]) :class:`xml.sax.handler.ContentHandler` implementation that ... diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index c8cc773..5954735 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -16,6 +16,14 @@ The :class:`Element` type is a flexible container object, designed to store hierarchical data structures in memory. The type can be described as a cross between a list and a dictionary. + +.. warning:: + + The :mod:`xml.etree.ElementTree` module is not secure against + maliciously constructed data. If you need to parse untrusted or + unauthenticated data see :ref:`xml-vulnerabilities`. + + Each element has a number of properties associated with it: * a tag which is a string identifying what kind of data this element represents diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst new file mode 100644 index 0000000..f84af58 --- /dev/null +++ b/Doc/library/xml.rst @@ -0,0 +1,131 @@ +.. _xml: + +XML Processing Modules +====================== + +.. module:: xml + :synopsis: Package containing XML processing modules +.. sectionauthor:: Christian Heimes +.. sectionauthor:: Georg Brandl + + +Python's interfaces for processing XML are grouped in the ``xml`` package. + +.. warning:: + + The XML modules are not secure against erroneous or maliciously + constructed data. If you need to parse untrusted or unauthenticated data see + :ref:`xml-vulnerabilities`. + +It is important to note that modules in the :mod:`xml` package require that +there be at least one SAX-compliant XML parser available. The Expat parser is +included with Python, so the :mod:`xml.parsers.expat` module will always be +available. + +The documentation for the :mod:`xml.dom` and :mod:`xml.sax` packages are the +definition of the Python bindings for the DOM and SAX interfaces. + +The XML handling submodules are: + +* :mod:`xml.etree.ElementTree`: the ElementTree API, a simple and lightweight + +.. + +* :mod:`xml.dom`: the DOM API definition +* :mod:`xml.dom.minidom`: a lightweight DOM implementation +* :mod:`xml.dom.pulldom`: support for building partial DOM trees + +.. + +* :mod:`xml.sax`: SAX2 base classes and convenience functions +* :mod:`xml.parsers.expat`: the Expat parser binding + + +.. _xml-vulnerabilities: + +XML vulnerabilities +=================== + +The XML processing modules are not secure against maliciously constructed data. +An attacker can abuse vulnerabilities for e.g. denial of service attacks, to +access local files, to generate network connections to other machines, or +to or circumvent firewalls. The attacks on XML abuse unfamiliar features +like inline `DTD`_ (document type definition) with entities. + + +========================= ======== ========= ========= ======== ========= +kind sax etree minidom pulldom xmlrpc +========================= ======== ========= ========= ======== ========= +billion laughs **True** **True** **True** **True** **True** +quadratic blowup **True** **True** **True** **True** **True** +external entity expansion **True** False (1) False (2) **True** False (3) +DTD retrieval **True** False False **True** False +decompression bomb False False False False **True** +========================= ======== ========= ========= ======== ========= + +1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a + ParserError when an entity occurs. +2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns + the unexpanded entity verbatim. +3. :mod:`xmlrpclib` doesn't expand external entities and omits them. + + +billion laughs / exponential entity expansion + The `Billion Laughs`_ attack -- also known as exponential entity expansion -- + uses multiple levels of nested entities. Each entity refers to another entity + several times, the final entity definition contains a small string. Eventually + the small string is expanded to several gigabytes. The exponential expansion + consumes lots of CPU time, too. + +quadratic blowup entity expansion + A quadratic blowup attack is similar to a `Billion Laughs`_ attack; it abuses + entity expansion, too. Instead of nested entities it repeats one large entity + with a couple of thousand chars over and over again. The attack isn't as + efficient as the exponential case but it avoids triggering countermeasures of + parsers against heavily nested entities. + +external entity expansion + Entity declarations can contain more than just text for replacement. They can + also point to external resources by public identifiers or system identifiers. + System identifiers are standard URIs or can refer to local files. The XML + parser retrieves the resource with e.g. HTTP or FTP requests and embeds the + content into the XML document. + +DTD retrieval + Some XML libraries like Python's mod:'xml.dom.pulldom' retrieve document type + definitions from remote or local locations. The feature has similar + implications as the external entity expansion issue. + +decompression bomb + The issue of decompression bombs (aka `ZIP bomb`_) apply to all XML libraries + that can parse compressed XML stream like gzipped HTTP streams or LZMA-ed + files. For an attacker it can reduce the amount of transmitted data by three + magnitudes or more. + +The documentation of `defusedxml`_ on PyPI has further information about +all known attack vectors with examples and references. + +defused packages +---------------- + +`defusedxml`_ is a pure Python package with modified subclasses of all stdlib +XML parsers that prevent any potentially malicious operation. The courses of +action are recommended for any server code that parses untrusted XML data. The +package also ships with example exploits and an extended documentation on more +XML exploits like xpath injection. + +`defusedexpat`_ provides a modified libexpat and patched replacment +:mod:`pyexpat` extension module with countermeasures against entity expansion +DoS attacks. Defusedexpat still allows a sane and configurable amount of entity +expansions. The modifications will be merged into future releases of Python. + +The workarounds and modifications are not included in patch releases as they +break backward compatibility. After all inline DTD and entity expansion are +well-definied XML features. + + +.. _defusedxml: +.. _defusedexpat: +.. _Billion Laughs: http://en.wikipedia.org/wiki/Billion_laughs +.. _ZIP bomb: http://en.wikipedia.org/wiki/Zip_bomb +.. _DTD: http://en.wikipedia.org/wiki/Document_Type_Definition diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 43d17c2..75cfc11 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -16,6 +16,14 @@ Simple API for XML (SAX) interface for Python. The package itself provides the SAX exceptions and the convenience functions which will be most used by users of the SAX API. + +.. warning:: + + The :mod:`xml.sax` module is not secure against maliciously + constructed data. If you need to parse untrusted or unauthenticated data see + :ref:`xml-vulnerabilities`. + + The convenience functions are: diff --git a/Doc/library/xmlrpclib.rst b/Doc/library/xmlrpclib.rst index f50f270..0e9ff4b 100644 --- a/Doc/library/xmlrpclib.rst +++ b/Doc/library/xmlrpclib.rst @@ -28,6 +28,13 @@ supports writing XML-RPC client code; it handles all the details of translating between conformable Python objects and XML on the wire. +.. warning:: + + The :mod:`xmlrpclib` module is not secure against maliciously + constructed data. If you need to parse untrusted or unauthenticated data see + :ref:`xml-vulnerabilities`. + + .. class:: ServerProxy(uri[, transport[, encoding[, verbose[, allow_none[, use_datetime]]]]]) A :class:`ServerProxy` instance is an object that manages communication with a diff --git a/Misc/NEWS b/Misc/NEWS index 2bbc633..adf78de 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,11 @@ Library - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. +Documentation +------------- + +- Issue 17538: Document XML vulnerabilties + What's New in Python 2.7.4 release candidate 1 ============================================== -- cgit v0.12 From bf607aa9da0c2a0c602bdfd397a606d06d9a2119 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 28 Mar 2013 11:42:49 +0100 Subject: Fix XML vulnerability link targets. --- Doc/library/xml.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index f84af58..a800813 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -124,8 +124,8 @@ break backward compatibility. After all inline DTD and entity expansion are well-definied XML features. -.. _defusedxml: -.. _defusedexpat: +.. _defusedxml: https://pypi.python.org/pypi/defusedxml/ +.. _defusedexpat: https://pypi.python.org/pypi/defusedexpat/ .. _Billion Laughs: http://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: http://en.wikipedia.org/wiki/Zip_bomb .. _DTD: http://en.wikipedia.org/wiki/Document_Type_Definition -- cgit v0.12 From 0861b13c981b93364db8a5bc184840acd4805497 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sat, 30 Mar 2013 01:38:38 -0700 Subject: Fix typos and clear up one very odd bit of wording as pointed out by Ezio. --- Doc/library/xml.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index a800813..88b8505 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -108,20 +108,22 @@ all known attack vectors with examples and references. defused packages ---------------- +These external packages are recommended for any code that parses +untrusted XML data. + `defusedxml`_ is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. The courses of -action are recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and an extended documentation on more +XML parsers that prevent any potentially malicious operation. The +package also ships with example exploits and extended documentation on more XML exploits like xpath injection. -`defusedexpat`_ provides a modified libexpat and patched replacment +`defusedexpat`_ provides a modified libexpat and patched replacement :mod:`pyexpat` extension module with countermeasures against entity expansion DoS attacks. Defusedexpat still allows a sane and configurable amount of entity expansions. The modifications will be merged into future releases of Python. The workarounds and modifications are not included in patch releases as they break backward compatibility. After all inline DTD and entity expansion are -well-definied XML features. +well-defined XML features. .. _defusedxml: https://pypi.python.org/pypi/defusedxml/ -- cgit v0.12 From d627e122d79802c3f91d6157af6af34d6f6a1757 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 30 Mar 2013 10:36:31 -0400 Subject: fall back when an old test_support doesn't have various data and functions (closes #17533) --- Lib/test/pickletester.py | 10 ++++++++-- Misc/NEWS | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 3e7e11d..c95531f 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -6,8 +6,14 @@ import cStringIO import pickletools import copy_reg -from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M, - precisionbigmemtest) +from test.test_support import TestFailed, verbose, have_unicode, TESTFN +try: + from test.test_support import _2G, _1M, precisionbigmemtest +except ImportError: + # this import might fail when run on older Python versions by test_xpickle + _2G = _1G = 0 + def precisionbigmemtest(*args, **kwargs): + return lambda self: None # Tests that try a number of pickle protocols should have a # for proto in protocols: diff --git a/Misc/NEWS b/Misc/NEWS index adf78de..4587309 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -24,6 +24,8 @@ Library - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. +- Issue #17533: Fix test_xpickle with older versions of Python 2.5. + Documentation ------------- -- cgit v0.12