From 4a247ed5e37d3b4a17c965c725b81f6ef835c191 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 2 Apr 2010 18:27:25 +0200 Subject: Update #include of private headers in QtDeclarative Always use private/. The WinSCW compiler doesn't search the current directory, for whatever reason. --- src/declarative/3rdparty/qlistmodelinterface.cpp | 2 +- src/declarative/debugger/qdeclarativedebug.cpp | 4 +- .../debugger/qdeclarativedebugclient.cpp | 4 +- .../debugger/qdeclarativedebuggerstatus.cpp | 2 +- .../debugger/qdeclarativedebugservice.cpp | 4 +- src/declarative/debugger/qpacketprotocol.cpp | 2 +- .../graphicsitems/qdeclarativeanchors.cpp | 4 +- .../graphicsitems/qdeclarativeanchors_p_p.h | 4 +- .../graphicsitems/qdeclarativeanimatedimage.cpp | 4 +- .../graphicsitems/qdeclarativeanimatedimage_p.h | 2 +- .../graphicsitems/qdeclarativeanimatedimage_p_p.h | 2 +- .../graphicsitems/qdeclarativeborderimage.cpp | 4 +- .../graphicsitems/qdeclarativeborderimage_p.h | 2 +- .../graphicsitems/qdeclarativeborderimage_p_p.h | 4 +- .../graphicsitems/qdeclarativeevents.cpp | 2 +- .../graphicsitems/qdeclarativeflickable.cpp | 4 +- .../graphicsitems/qdeclarativeflickable_p_p.h | 6 +- .../graphicsitems/qdeclarativeflipable.cpp | 6 +- .../graphicsitems/qdeclarativefocuspanel.cpp | 4 +- .../graphicsitems/qdeclarativefocusscope.cpp | 4 +- .../qdeclarativegraphicsobjectcontainer.cpp | 4 +- .../graphicsitems/qdeclarativegridview.cpp | 8 +-- .../graphicsitems/qdeclarativegridview_p.h | 2 +- .../graphicsitems/qdeclarativeimage.cpp | 4 +- .../graphicsitems/qdeclarativeimage_p.h | 2 +- .../graphicsitems/qdeclarativeimage_p_p.h | 4 +- .../graphicsitems/qdeclarativeimagebase.cpp | 4 +- .../graphicsitems/qdeclarativeimagebase_p_p.h | 2 +- src/declarative/graphicsitems/qdeclarativeitem.cpp | 4 +- .../graphicsitems/qdeclarativeitemsmodule.cpp | 66 +++++++++++----------- .../graphicsitems/qdeclarativelayoutitem.cpp | 2 +- .../graphicsitems/qdeclarativelistview.cpp | 8 +-- .../graphicsitems/qdeclarativelistview_p.h | 2 +- .../graphicsitems/qdeclarativeloader.cpp | 2 +- .../graphicsitems/qdeclarativeloader_p_p.h | 6 +- .../graphicsitems/qdeclarativemousearea.cpp | 6 +- .../graphicsitems/qdeclarativemousearea_p_p.h | 2 +- .../graphicsitems/qdeclarativepainteditem.cpp | 4 +- .../graphicsitems/qdeclarativepainteditem_p_p.h | 2 +- src/declarative/graphicsitems/qdeclarativepath.cpp | 4 +- .../graphicsitems/qdeclarativepath_p_p.h | 2 +- .../graphicsitems/qdeclarativepathview.cpp | 4 +- .../graphicsitems/qdeclarativepathview_p.h | 2 +- .../graphicsitems/qdeclarativepathview_p_p.h | 6 +- .../graphicsitems/qdeclarativepositioners.cpp | 4 +- .../graphicsitems/qdeclarativepositioners_p_p.h | 4 +- .../graphicsitems/qdeclarativerectangle.cpp | 4 +- .../graphicsitems/qdeclarativerectangle_p_p.h | 2 +- .../graphicsitems/qdeclarativerepeater.cpp | 6 +- .../graphicsitems/qdeclarativerepeater_p_p.h | 4 +- .../graphicsitems/qdeclarativescalegrid.cpp | 2 +- .../graphicsitems/qdeclarativescalegrid_p_p.h | 2 +- src/declarative/graphicsitems/qdeclarativetext.cpp | 4 +- .../graphicsitems/qdeclarativetext_p_p.h | 2 +- .../graphicsitems/qdeclarativetextedit.cpp | 6 +- .../graphicsitems/qdeclarativetextedit_p.h | 4 +- .../graphicsitems/qdeclarativetextedit_p_p.h | 2 +- .../graphicsitems/qdeclarativetextinput.cpp | 4 +- .../graphicsitems/qdeclarativetextinput_p.h | 4 +- .../graphicsitems/qdeclarativetextinput_p_p.h | 4 +- .../graphicsitems/qdeclarativetranslate.cpp | 2 +- .../graphicsitems/qdeclarativevisualitemmodel.cpp | 2 +- src/declarative/qml/parser/qdeclarativejs.g | 18 +++--- src/declarative/qml/parser/qdeclarativejsast.cpp | 4 +- src/declarative/qml/parser/qdeclarativejsast_p.h | 4 +- .../qml/parser/qdeclarativejsastfwd_p.h | 2 +- .../qml/parser/qdeclarativejsastvisitor.cpp | 2 +- .../qml/parser/qdeclarativejsastvisitor_p.h | 4 +- .../qml/parser/qdeclarativejsengine_p.cpp | 6 +- .../qml/parser/qdeclarativejsengine_p.h | 4 +- .../qml/parser/qdeclarativejsgrammar.cpp | 2 +- src/declarative/qml/parser/qdeclarativejslexer.cpp | 8 +-- src/declarative/qml/parser/qdeclarativejslexer_p.h | 2 +- .../qml/parser/qdeclarativejsmemorypool_p.h | 2 +- .../qml/parser/qdeclarativejsnodepool_p.h | 4 +- .../qml/parser/qdeclarativejsparser.cpp | 10 ++-- .../qml/parser/qdeclarativejsparser_p.h | 8 +-- src/declarative/qml/qdeclarativebinding.cpp | 10 ++-- src/declarative/qml/qdeclarativebinding_p.h | 2 +- src/declarative/qml/qdeclarativebinding_p_p.h | 4 +- src/declarative/qml/qdeclarativeboundsignal.cpp | 14 ++--- src/declarative/qml/qdeclarativeclassfactory.cpp | 2 +- src/declarative/qml/qdeclarativecleanup.cpp | 4 +- .../qml/qdeclarativecompiledbindings.cpp | 2 +- .../qml/qdeclarativecompiledbindings_p.h | 4 +- src/declarative/qml/qdeclarativecompileddata.cpp | 6 +- src/declarative/qml/qdeclarativecompiler.cpp | 40 ++++++------- src/declarative/qml/qdeclarativecompiler_p.h | 16 +++--- src/declarative/qml/qdeclarativecomponent.cpp | 20 +++---- src/declarative/qml/qdeclarativecomponent_p.h | 6 +- .../qml/qdeclarativecompositetypedata_p.h | 2 +- .../qml/qdeclarativecompositetypemanager.cpp | 12 ++-- .../qml/qdeclarativecompositetypemanager_p.h | 4 +- src/declarative/qml/qdeclarativecontext.cpp | 10 ++-- src/declarative/qml/qdeclarativecontext_p.h | 12 ++-- .../qml/qdeclarativecontextscriptclass.cpp | 12 ++-- .../qml/qdeclarativecontextscriptclass_p.h | 4 +- src/declarative/qml/qdeclarativecustomparser.cpp | 8 +-- src/declarative/qml/qdeclarativecustomparser_p.h | 4 +- src/declarative/qml/qdeclarativecustomparser_p_p.h | 4 +- .../qml/qdeclarativedeclarativedata_p.h | 2 +- src/declarative/qml/qdeclarativedirparser.cpp | 2 +- src/declarative/qml/qdeclarativedom.cpp | 16 +++--- src/declarative/qml/qdeclarativedom_p_p.h | 2 +- src/declarative/qml/qdeclarativeengine.cpp | 38 ++++++------- src/declarative/qml/qdeclarativeengine_p.h | 24 ++++---- src/declarative/qml/qdeclarativeenginedebug.cpp | 16 +++--- src/declarative/qml/qdeclarativeexpression.cpp | 12 ++-- src/declarative/qml/qdeclarativeexpression_p.h | 4 +- src/declarative/qml/qdeclarativefastproperties.cpp | 2 +- .../qml/qdeclarativeglobalscriptclass.cpp | 2 +- src/declarative/qml/qdeclarativeguard_p.h | 2 +- src/declarative/qml/qdeclarativeinfo.cpp | 6 +- src/declarative/qml/qdeclarativeinstruction.cpp | 4 +- src/declarative/qml/qdeclarativeintegercache.cpp | 6 +- src/declarative/qml/qdeclarativeintegercache_p.h | 4 +- src/declarative/qml/qdeclarativelist.cpp | 6 +- src/declarative/qml/qdeclarativelist.h | 2 +- src/declarative/qml/qdeclarativelist_p.h | 2 +- .../qml/qdeclarativelistscriptclass.cpp | 8 +-- src/declarative/qml/qdeclarativemetatype.cpp | 8 +-- src/declarative/qml/qdeclarativenotifier.cpp | 2 +- src/declarative/qml/qdeclarativenotifier_p.h | 2 +- .../qml/qdeclarativeobjectscriptclass.cpp | 20 +++---- .../qml/qdeclarativeobjectscriptclass_p.h | 4 +- src/declarative/qml/qdeclarativeparser.cpp | 12 ++-- src/declarative/qml/qdeclarativeparser_p.h | 2 +- src/declarative/qml/qdeclarativeproperty.cpp | 20 +++---- src/declarative/qml/qdeclarativeproperty_p.h | 4 +- src/declarative/qml/qdeclarativepropertycache.cpp | 6 +- src/declarative/qml/qdeclarativepropertycache_p.h | 6 +- .../qml/qdeclarativeproxymetaobject.cpp | 2 +- .../qml/qdeclarativeproxymetaobject_p.h | 2 +- src/declarative/qml/qdeclarativerefcount.cpp | 2 +- src/declarative/qml/qdeclarativerewrite.cpp | 4 +- src/declarative/qml/qdeclarativescriptparser.cpp | 6 +- src/declarative/qml/qdeclarativescriptparser_p.h | 2 +- src/declarative/qml/qdeclarativesqldatabase.cpp | 8 +-- .../qml/qdeclarativestringconverters.cpp | 2 +- src/declarative/qml/qdeclarativetypenamecache.cpp | 4 +- src/declarative/qml/qdeclarativetypenamecache_p.h | 4 +- .../qml/qdeclarativetypenamescriptclass.cpp | 6 +- .../qml/qdeclarativetypenamescriptclass_p.h | 2 +- src/declarative/qml/qdeclarativevaluetype.cpp | 4 +- src/declarative/qml/qdeclarativevaluetype_p.h | 2 +- .../qml/qdeclarativevaluetypescriptclass.cpp | 10 ++-- src/declarative/qml/qdeclarativevme.cpp | 30 +++++----- src/declarative/qml/qdeclarativevme_p.h | 2 +- src/declarative/qml/qdeclarativevmemetaobject.cpp | 8 +-- src/declarative/qml/qdeclarativevmemetaobject_p.h | 6 +- src/declarative/qml/qdeclarativewatcher.cpp | 2 +- src/declarative/qml/qdeclarativeworkerscript.cpp | 8 +-- src/declarative/qml/qdeclarativexmlhttprequest.cpp | 10 ++-- src/declarative/qml/qmetaobjectbuilder.cpp | 2 +- src/declarative/qml/rewriter/rewriter.cpp | 2 +- src/declarative/qml/rewriter/rewriter_p.h | 2 +- src/declarative/qml/rewriter/textwriter.cpp | 2 +- src/declarative/util/qdeclarativeanimation.cpp | 8 +-- src/declarative/util/qdeclarativeanimation_p.h | 4 +- src/declarative/util/qdeclarativeanimation_p_p.h | 6 +- src/declarative/util/qdeclarativebehavior.cpp | 6 +- src/declarative/util/qdeclarativebehavior_p.h | 2 +- src/declarative/util/qdeclarativebind.cpp | 4 +- src/declarative/util/qdeclarativeconnections.cpp | 2 +- src/declarative/util/qdeclarativefontloader.cpp | 2 +- src/declarative/util/qdeclarativelistaccessor.cpp | 2 +- src/declarative/util/qdeclarativelistmodel.cpp | 6 +- src/declarative/util/qdeclarativelistmodel_p_p.h | 4 +- .../util/qdeclarativelistmodelworkeragent.cpp | 8 +-- .../util/qdeclarativeopenmetaobject.cpp | 6 +- src/declarative/util/qdeclarativepackage.cpp | 2 +- src/declarative/util/qdeclarativepixmapcache.cpp | 2 +- .../util/qdeclarativepropertychanges.cpp | 4 +- .../util/qdeclarativepropertychanges_p.h | 2 +- src/declarative/util/qdeclarativepropertymap.cpp | 2 +- .../util/qdeclarativesmoothedanimation.cpp | 10 ++-- .../util/qdeclarativesmoothedanimation_p.h | 2 +- .../util/qdeclarativesmoothedanimation_p_p.h | 6 +- src/declarative/util/qdeclarativespringfollow.cpp | 4 +- src/declarative/util/qdeclarativestate.cpp | 16 +++--- src/declarative/util/qdeclarativestate_p_p.h | 6 +- src/declarative/util/qdeclarativestategroup.cpp | 6 +- src/declarative/util/qdeclarativestategroup_p.h | 2 +- .../util/qdeclarativestateoperations.cpp | 2 +- .../util/qdeclarativestateoperations_p.h | 2 +- src/declarative/util/qdeclarativestyledtext.cpp | 2 +- src/declarative/util/qdeclarativesystempalette.cpp | 2 +- src/declarative/util/qdeclarativetimeline.cpp | 2 +- src/declarative/util/qdeclarativetimer.cpp | 2 +- src/declarative/util/qdeclarativetransition.cpp | 14 ++--- src/declarative/util/qdeclarativetransition_p.h | 2 +- .../util/qdeclarativetransitionmanager.cpp | 4 +- .../util/qdeclarativetransitionmanager_p_p.h | 2 +- src/declarative/util/qdeclarativeutilmodule.cpp | 54 +++++++++--------- src/declarative/util/qdeclarativexmllistmodel.cpp | 2 +- 195 files changed, 580 insertions(+), 580 deletions(-) diff --git a/src/declarative/3rdparty/qlistmodelinterface.cpp b/src/declarative/3rdparty/qlistmodelinterface.cpp index 50714ce..98d6a5b 100644 --- a/src/declarative/3rdparty/qlistmodelinterface.cpp +++ b/src/declarative/3rdparty/qlistmodelinterface.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qlistmodelinterface_p.h" +#include "private/qlistmodelinterface_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/debugger/qdeclarativedebug.cpp b/src/declarative/debugger/qdeclarativedebug.cpp index 677d05f..cd5340b 100644 --- a/src/declarative/debugger/qdeclarativedebug.cpp +++ b/src/declarative/debugger/qdeclarativedebug.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativedebug_p.h" +#include "private/qdeclarativedebug_p.h" -#include "qdeclarativedebugclient_p.h" +#include "private/qdeclarativedebugclient_p.h" #include diff --git a/src/declarative/debugger/qdeclarativedebugclient.cpp b/src/declarative/debugger/qdeclarativedebugclient.cpp index c23e32f..2e52b40 100644 --- a/src/declarative/debugger/qdeclarativedebugclient.cpp +++ b/src/declarative/debugger/qdeclarativedebugclient.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativedebugclient_p.h" +#include "private/qdeclarativedebugclient_p.h" -#include "qpacketprotocol_p.h" +#include "private/qpacketprotocol_p.h" #include #include diff --git a/src/declarative/debugger/qdeclarativedebuggerstatus.cpp b/src/declarative/debugger/qdeclarativedebuggerstatus.cpp index 5908628..476cca2 100644 --- a/src/declarative/debugger/qdeclarativedebuggerstatus.cpp +++ b/src/declarative/debugger/qdeclarativedebuggerstatus.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativedebuggerstatus_p.h" +#include "private/qdeclarativedebuggerstatus_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/debugger/qdeclarativedebugservice.cpp b/src/declarative/debugger/qdeclarativedebugservice.cpp index d9bbdb5..9d9d1d0 100644 --- a/src/declarative/debugger/qdeclarativedebugservice.cpp +++ b/src/declarative/debugger/qdeclarativedebugservice.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativedebugservice_p.h" +#include "private/qdeclarativedebugservice_p.h" -#include "qpacketprotocol_p.h" +#include "private/qpacketprotocol_p.h" #include #include diff --git a/src/declarative/debugger/qpacketprotocol.cpp b/src/declarative/debugger/qpacketprotocol.cpp index 7440b87..6241922 100644 --- a/src/declarative/debugger/qpacketprotocol.cpp +++ b/src/declarative/debugger/qpacketprotocol.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qpacketprotocol_p.h" +#include "private/qpacketprotocol_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeanchors.cpp b/src/declarative/graphicsitems/qdeclarativeanchors.cpp index ffcedda..7a7e5be 100644 --- a/src/declarative/graphicsitems/qdeclarativeanchors.cpp +++ b/src/declarative/graphicsitems/qdeclarativeanchors.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativeanchors_p_p.h" +#include "private/qdeclarativeanchors_p_p.h" #include "qdeclarativeitem.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeanchors_p_p.h b/src/declarative/graphicsitems/qdeclarativeanchors_p_p.h index 5840868..ba6359a 100644 --- a/src/declarative/graphicsitems/qdeclarativeanchors_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeanchors_p_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativeanchors_p.h" -#include "qdeclarativeitemchangelistener_p.h" +#include "private/qdeclarativeanchors_p.h" +#include "private/qdeclarativeitemchangelistener_p.h" #include QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativeanimatedimage.cpp b/src/declarative/graphicsitems/qdeclarativeanimatedimage.cpp index 2c2e034..f14f773 100644 --- a/src/declarative/graphicsitems/qdeclarativeanimatedimage.cpp +++ b/src/declarative/graphicsitems/qdeclarativeanimatedimage.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativeanimatedimage_p.h" -#include "qdeclarativeanimatedimage_p_p.h" +#include "private/qdeclarativeanimatedimage_p.h" +#include "private/qdeclarativeanimatedimage_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeanimatedimage_p.h b/src/declarative/graphicsitems/qdeclarativeanimatedimage_p.h index b2979fe..6ab66b3 100644 --- a/src/declarative/graphicsitems/qdeclarativeanimatedimage_p.h +++ b/src/declarative/graphicsitems/qdeclarativeanimatedimage_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEANIMATEDIMAGE_H #define QDECLARATIVEANIMATEDIMAGE_H -#include "qdeclarativeimage_p.h" +#include "private/qdeclarativeimage_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/graphicsitems/qdeclarativeanimatedimage_p_p.h b/src/declarative/graphicsitems/qdeclarativeanimatedimage_p_p.h index 273c1d6..8ca9755 100644 --- a/src/declarative/graphicsitems/qdeclarativeanimatedimage_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeanimatedimage_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeimage_p_p.h" +#include "private/qdeclarativeimage_p_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativeborderimage.cpp b/src/declarative/graphicsitems/qdeclarativeborderimage.cpp index 96f95f2..420ed90 100644 --- a/src/declarative/graphicsitems/qdeclarativeborderimage.cpp +++ b/src/declarative/graphicsitems/qdeclarativeborderimage.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativeborderimage_p.h" -#include "qdeclarativeborderimage_p_p.h" +#include "private/qdeclarativeborderimage_p.h" +#include "private/qdeclarativeborderimage_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeborderimage_p.h b/src/declarative/graphicsitems/qdeclarativeborderimage_p.h index a759e67..5e725ca 100644 --- a/src/declarative/graphicsitems/qdeclarativeborderimage_p.h +++ b/src/declarative/graphicsitems/qdeclarativeborderimage_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEBORDERIMAGE_H #define QDECLARATIVEBORDERIMAGE_H -#include "qdeclarativeimagebase_p.h" +#include "private/qdeclarativeimagebase_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeborderimage_p_p.h b/src/declarative/graphicsitems/qdeclarativeborderimage_p_p.h index 82b9ebf..3535109 100644 --- a/src/declarative/graphicsitems/qdeclarativeborderimage_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeborderimage_p_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativeimagebase_p_p.h" -#include "qdeclarativescalegrid_p_p.h" +#include "private/qdeclarativeimagebase_p_p.h" +#include "private/qdeclarativescalegrid_p_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativeevents.cpp b/src/declarative/graphicsitems/qdeclarativeevents.cpp index 8be2f40..6118ea8 100644 --- a/src/declarative/graphicsitems/qdeclarativeevents.cpp +++ b/src/declarative/graphicsitems/qdeclarativeevents.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeevents_p_p.h" +#include "private/qdeclarativeevents_p_p.h" QT_BEGIN_NAMESPACE /*! diff --git a/src/declarative/graphicsitems/qdeclarativeflickable.cpp b/src/declarative/graphicsitems/qdeclarativeflickable.cpp index 8459245..fc7a87b 100644 --- a/src/declarative/graphicsitems/qdeclarativeflickable.cpp +++ b/src/declarative/graphicsitems/qdeclarativeflickable.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativeflickable_p.h" -#include "qdeclarativeflickable_p_p.h" +#include "private/qdeclarativeflickable_p.h" +#include "private/qdeclarativeflickable_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeflickable_p_p.h b/src/declarative/graphicsitems/qdeclarativeflickable_p_p.h index c963c2b..1a04091 100644 --- a/src/declarative/graphicsitems/qdeclarativeflickable_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeflickable_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativeflickable_p.h" +#include "private/qdeclarativeflickable_p.h" -#include "qdeclarativeitem_p.h" -#include "qdeclarativeitemchangelistener_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativeitemchangelistener_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeflipable.cpp b/src/declarative/graphicsitems/qdeclarativeflipable.cpp index ccefc70..98e34a9 100644 --- a/src/declarative/graphicsitems/qdeclarativeflipable.cpp +++ b/src/declarative/graphicsitems/qdeclarativeflipable.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativeflipable_p.h" +#include "private/qdeclarativeflipable_p.h" -#include "qdeclarativeitem_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativeguard_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativefocuspanel.cpp b/src/declarative/graphicsitems/qdeclarativefocuspanel.cpp index 20524b6..bd60e25 100644 --- a/src/declarative/graphicsitems/qdeclarativefocuspanel.cpp +++ b/src/declarative/graphicsitems/qdeclarativefocuspanel.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativefocuspanel_p.h" +#include "private/qdeclarativefocuspanel_p.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativefocusscope.cpp b/src/declarative/graphicsitems/qdeclarativefocusscope.cpp index 484df13..4bd2a86 100644 --- a/src/declarative/graphicsitems/qdeclarativefocusscope.cpp +++ b/src/declarative/graphicsitems/qdeclarativefocusscope.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativefocusscope_p.h" +#include "private/qdeclarativefocusscope_p.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativegraphicsobjectcontainer.cpp b/src/declarative/graphicsitems/qdeclarativegraphicsobjectcontainer.cpp index ff85bbd..c846431 100644 --- a/src/declarative/graphicsitems/qdeclarativegraphicsobjectcontainer.cpp +++ b/src/declarative/graphicsitems/qdeclarativegraphicsobjectcontainer.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativegraphicsobjectcontainer_p.h" +#include "private/qdeclarativegraphicsobjectcontainer_p.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativegridview.cpp b/src/declarative/graphicsitems/qdeclarativegridview.cpp index 30f04f6..8247f17 100644 --- a/src/declarative/graphicsitems/qdeclarativegridview.cpp +++ b/src/declarative/graphicsitems/qdeclarativegridview.cpp @@ -39,12 +39,12 @@ ** ****************************************************************************/ -#include "qdeclarativegridview_p.h" +#include "private/qdeclarativegridview_p.h" -#include "qdeclarativevisualitemmodel_p.h" -#include "qdeclarativeflickable_p_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativeflickable_p_p.h" -#include "qdeclarativesmoothedanimation_p_p.h" +#include "private/qdeclarativesmoothedanimation_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativegridview_p.h b/src/declarative/graphicsitems/qdeclarativegridview_p.h index 90f13d2..f73f632 100644 --- a/src/declarative/graphicsitems/qdeclarativegridview_p.h +++ b/src/declarative/graphicsitems/qdeclarativegridview_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEGRIDVIEW_H #define QDECLARATIVEGRIDVIEW_H -#include "qdeclarativeflickable_p.h" +#include "private/qdeclarativeflickable_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/graphicsitems/qdeclarativeimage.cpp b/src/declarative/graphicsitems/qdeclarativeimage.cpp index 3145c43..e4f3464 100644 --- a/src/declarative/graphicsitems/qdeclarativeimage.cpp +++ b/src/declarative/graphicsitems/qdeclarativeimage.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativeimage_p.h" -#include "qdeclarativeimage_p_p.h" +#include "private/qdeclarativeimage_p.h" +#include "private/qdeclarativeimage_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeimage_p.h b/src/declarative/graphicsitems/qdeclarativeimage_p.h index da6cbd5..5ea700d 100644 --- a/src/declarative/graphicsitems/qdeclarativeimage_p.h +++ b/src/declarative/graphicsitems/qdeclarativeimage_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEIMAGE_H #define QDECLARATIVEIMAGE_H -#include "qdeclarativeimagebase_p.h" +#include "private/qdeclarativeimagebase_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeimage_p_p.h b/src/declarative/graphicsitems/qdeclarativeimage_p_p.h index 8102237..dd12a00 100644 --- a/src/declarative/graphicsitems/qdeclarativeimage_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeimage_p_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativeitem_p.h" -#include "qdeclarativeimagebase_p_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativeimagebase_p_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativeimagebase.cpp b/src/declarative/graphicsitems/qdeclarativeimagebase.cpp index 5a234b7..3acafe8 100644 --- a/src/declarative/graphicsitems/qdeclarativeimagebase.cpp +++ b/src/declarative/graphicsitems/qdeclarativeimagebase.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativeimagebase_p.h" -#include "qdeclarativeimagebase_p_p.h" +#include "private/qdeclarativeimagebase_p.h" +#include "private/qdeclarativeimagebase_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeimagebase_p_p.h b/src/declarative/graphicsitems/qdeclarativeimagebase_p_p.h index de8c93a..392c1db 100644 --- a/src/declarative/graphicsitems/qdeclarativeimagebase_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeimagebase_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativeitem.cpp b/src/declarative/graphicsitems/qdeclarativeitem.cpp index 83c79a0..55a81f4 100644 --- a/src/declarative/graphicsitems/qdeclarativeitem.cpp +++ b/src/declarative/graphicsitems/qdeclarativeitem.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include "qdeclarativeitem.h" -#include "qdeclarativeevents_p_p.h" +#include "private/qdeclarativeevents_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeitemsmodule.cpp b/src/declarative/graphicsitems/qdeclarativeitemsmodule.cpp index 97a22cf..7989a27 100644 --- a/src/declarative/graphicsitems/qdeclarativeitemsmodule.cpp +++ b/src/declarative/graphicsitems/qdeclarativeitemsmodule.cpp @@ -39,46 +39,46 @@ ** ****************************************************************************/ -#include "qdeclarativeitemsmodule_p.h" +#include "private/qdeclarativeitemsmodule_p.h" #include #include #include -#include "qdeclarativeevents_p_p.h" -#include "qdeclarativeeffects_p.h" -#include "qdeclarativescalegrid_p_p.h" -#include "qdeclarativeanimatedimage_p.h" -#include "qdeclarativeborderimage_p.h" -#include "qdeclarativepositioners_p.h" -#include "qdeclarativemousearea_p.h" -#include "qdeclarativeflickable_p.h" -#include "qdeclarativeflickable_p_p.h" -#include "qdeclarativeflipable_p.h" -#include "qdeclarativefocuspanel_p.h" -#include "qdeclarativefocusscope_p.h" -#include "qdeclarativegraphicsobjectcontainer_p.h" -#include "qdeclarativegridview_p.h" -#include "qdeclarativeimage_p.h" -#include "qdeclarativeitem_p.h" -#include "qdeclarativelayoutitem_p.h" -#include "qdeclarativelistview_p.h" -#include "qdeclarativeloader_p.h" -#include "qdeclarativemousearea_p.h" -#include "qdeclarativepath_p.h" -#include "qdeclarativepathview_p.h" -#include "qdeclarativerectangle_p.h" -#include "qdeclarativerepeater_p.h" -#include "qdeclarativetranslate_p.h" -#include "qdeclarativetext_p.h" -#include "qdeclarativetextedit_p.h" -#include "qdeclarativetextinput_p.h" -#include "qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativeevents_p_p.h" +#include "private/qdeclarativeeffects_p.h" +#include "private/qdeclarativescalegrid_p_p.h" +#include "private/qdeclarativeanimatedimage_p.h" +#include "private/qdeclarativeborderimage_p.h" +#include "private/qdeclarativepositioners_p.h" +#include "private/qdeclarativemousearea_p.h" +#include "private/qdeclarativeflickable_p.h" +#include "private/qdeclarativeflickable_p_p.h" +#include "private/qdeclarativeflipable_p.h" +#include "private/qdeclarativefocuspanel_p.h" +#include "private/qdeclarativefocusscope_p.h" +#include "private/qdeclarativegraphicsobjectcontainer_p.h" +#include "private/qdeclarativegridview_p.h" +#include "private/qdeclarativeimage_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativelayoutitem_p.h" +#include "private/qdeclarativelistview_p.h" +#include "private/qdeclarativeloader_p.h" +#include "private/qdeclarativemousearea_p.h" +#include "private/qdeclarativepath_p.h" +#include "private/qdeclarativepathview_p.h" +#include "private/qdeclarativerectangle_p.h" +#include "private/qdeclarativerepeater_p.h" +#include "private/qdeclarativetranslate_p.h" +#include "private/qdeclarativetext_p.h" +#include "private/qdeclarativetextedit_p.h" +#include "private/qdeclarativetextinput_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" #ifdef QT_WEBKIT_LIB -#include "qdeclarativewebview_p.h" -#include "qdeclarativewebview_p_p.h" +#include "private/qdeclarativewebview_p.h" +#include "private/qdeclarativewebview_p_p.h" #endif -#include "qdeclarativeanchors_p.h" +#include "private/qdeclarativeanchors_p.h" void QDeclarativeItemModule::defineModule() { diff --git a/src/declarative/graphicsitems/qdeclarativelayoutitem.cpp b/src/declarative/graphicsitems/qdeclarativelayoutitem.cpp index a23ea65..1bbdd97 100644 --- a/src/declarative/graphicsitems/qdeclarativelayoutitem.cpp +++ b/src/declarative/graphicsitems/qdeclarativelayoutitem.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativelayoutitem_p.h" +#include "private/qdeclarativelayoutitem_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativelistview.cpp b/src/declarative/graphicsitems/qdeclarativelistview.cpp index 308612f..e85d60f 100644 --- a/src/declarative/graphicsitems/qdeclarativelistview.cpp +++ b/src/declarative/graphicsitems/qdeclarativelistview.cpp @@ -39,12 +39,12 @@ ** ****************************************************************************/ -#include "qdeclarativelistview_p.h" +#include "private/qdeclarativelistview_p.h" -#include "qdeclarativeflickable_p_p.h" -#include "qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativeflickable_p_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" -#include "qdeclarativesmoothedanimation_p_p.h" +#include "private/qdeclarativesmoothedanimation_p_p.h" #include #include #include diff --git a/src/declarative/graphicsitems/qdeclarativelistview_p.h b/src/declarative/graphicsitems/qdeclarativelistview_p.h index 3635b39..5810979 100644 --- a/src/declarative/graphicsitems/qdeclarativelistview_p.h +++ b/src/declarative/graphicsitems/qdeclarativelistview_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVELISTVIEW_H #define QDECLARATIVELISTVIEW_H -#include "qdeclarativeflickable_p.h" +#include "private/qdeclarativeflickable_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/graphicsitems/qdeclarativeloader.cpp b/src/declarative/graphicsitems/qdeclarativeloader.cpp index 2f1511e..2aab36f 100644 --- a/src/declarative/graphicsitems/qdeclarativeloader.cpp +++ b/src/declarative/graphicsitems/qdeclarativeloader.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeloader_p_p.h" +#include "private/qdeclarativeloader_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativeloader_p_p.h b/src/declarative/graphicsitems/qdeclarativeloader_p_p.h index fc5e665..49069f9 100644 --- a/src/declarative/graphicsitems/qdeclarativeloader_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativeloader_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativeloader_p.h" +#include "private/qdeclarativeloader_p.h" -#include "qdeclarativeitem_p.h" -#include "qdeclarativeitemchangelistener_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativeitemchangelistener_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativemousearea.cpp b/src/declarative/graphicsitems/qdeclarativemousearea.cpp index 6126a6f..816aa78 100644 --- a/src/declarative/graphicsitems/qdeclarativemousearea.cpp +++ b/src/declarative/graphicsitems/qdeclarativemousearea.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativemousearea_p.h" -#include "qdeclarativemousearea_p_p.h" +#include "private/qdeclarativemousearea_p.h" +#include "private/qdeclarativemousearea_p_p.h" -#include "qdeclarativeevents_p_p.h" +#include "private/qdeclarativeevents_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativemousearea_p_p.h b/src/declarative/graphicsitems/qdeclarativemousearea_p_p.h index 88206cd..9068c7c 100644 --- a/src/declarative/graphicsitems/qdeclarativemousearea_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativemousearea_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepainteditem.cpp b/src/declarative/graphicsitems/qdeclarativepainteditem.cpp index ab6007a..f52636f 100644 --- a/src/declarative/graphicsitems/qdeclarativepainteditem.cpp +++ b/src/declarative/graphicsitems/qdeclarativepainteditem.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativepainteditem_p.h" -#include "qdeclarativepainteditem_p_p.h" +#include "private/qdeclarativepainteditem_p.h" +#include "private/qdeclarativepainteditem_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepainteditem_p_p.h b/src/declarative/graphicsitems/qdeclarativepainteditem_p_p.h index a15febb..6b23086 100644 --- a/src/declarative/graphicsitems/qdeclarativepainteditem_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativepainteditem_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativepath.cpp b/src/declarative/graphicsitems/qdeclarativepath.cpp index 8cd990fc..e2042fc 100644 --- a/src/declarative/graphicsitems/qdeclarativepath.cpp +++ b/src/declarative/graphicsitems/qdeclarativepath.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativepath_p.h" -#include "qdeclarativepath_p_p.h" +#include "private/qdeclarativepath_p.h" +#include "private/qdeclarativepath_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepath_p_p.h b/src/declarative/graphicsitems/qdeclarativepath_p_p.h index fb63867..e82bcf5 100644 --- a/src/declarative/graphicsitems/qdeclarativepath_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativepath_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativepath_p.h" +#include "private/qdeclarativepath_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativepathview.cpp b/src/declarative/graphicsitems/qdeclarativepathview.cpp index 71f85ae..813b525 100644 --- a/src/declarative/graphicsitems/qdeclarativepathview.cpp +++ b/src/declarative/graphicsitems/qdeclarativepathview.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativepathview_p.h" -#include "qdeclarativepathview_p_p.h" +#include "private/qdeclarativepathview_p.h" +#include "private/qdeclarativepathview_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepathview_p.h b/src/declarative/graphicsitems/qdeclarativepathview_p.h index a00f4c0..69770cd 100644 --- a/src/declarative/graphicsitems/qdeclarativepathview_p.h +++ b/src/declarative/graphicsitems/qdeclarativepathview_p.h @@ -43,7 +43,7 @@ #define QDECLARATIVEPATHVIEW_H #include "qdeclarativeitem.h" -#include "qdeclarativepath_p.h" +#include "private/qdeclarativepath_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/graphicsitems/qdeclarativepathview_p_p.h b/src/declarative/graphicsitems/qdeclarativepathview_p_p.h index 6470893..11712fd 100644 --- a/src/declarative/graphicsitems/qdeclarativepathview_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativepathview_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativepathview_p.h" +#include "private/qdeclarativepathview_p.h" -#include "qdeclarativeitem_p.h" -#include "qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativeitem_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepositioners.cpp b/src/declarative/graphicsitems/qdeclarativepositioners.cpp index 781e584..d33a8be 100644 --- a/src/declarative/graphicsitems/qdeclarativepositioners.cpp +++ b/src/declarative/graphicsitems/qdeclarativepositioners.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativepositioners_p.h" -#include "qdeclarativepositioners_p_p.h" +#include "private/qdeclarativepositioners_p.h" +#include "private/qdeclarativepositioners_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativepositioners_p_p.h b/src/declarative/graphicsitems/qdeclarativepositioners_p_p.h index 7880e3e..576f35b 100644 --- a/src/declarative/graphicsitems/qdeclarativepositioners_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativepositioners_p_p.h @@ -53,9 +53,9 @@ // We mean it. // -#include "qdeclarativepositioners_p.h" +#include "private/qdeclarativepositioners_p.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativerectangle.cpp b/src/declarative/graphicsitems/qdeclarativerectangle.cpp index 8fbaa22..3f7548f 100644 --- a/src/declarative/graphicsitems/qdeclarativerectangle.cpp +++ b/src/declarative/graphicsitems/qdeclarativerectangle.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativerectangle_p.h" -#include "qdeclarativerectangle_p_p.h" +#include "private/qdeclarativerectangle_p.h" +#include "private/qdeclarativerectangle_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativerectangle_p_p.h b/src/declarative/graphicsitems/qdeclarativerectangle_p_p.h index 6bae219..84418bc 100644 --- a/src/declarative/graphicsitems/qdeclarativerectangle_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativerectangle_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/graphicsitems/qdeclarativerepeater.cpp b/src/declarative/graphicsitems/qdeclarativerepeater.cpp index bbee4e5..e836788 100644 --- a/src/declarative/graphicsitems/qdeclarativerepeater.cpp +++ b/src/declarative/graphicsitems/qdeclarativerepeater.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativerepeater_p.h" -#include "qdeclarativerepeater_p_p.h" +#include "private/qdeclarativerepeater_p.h" +#include "private/qdeclarativerepeater_p_p.h" -#include "qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativerepeater_p_p.h b/src/declarative/graphicsitems/qdeclarativerepeater_p_p.h index 11773ff..5b75e50 100644 --- a/src/declarative/graphicsitems/qdeclarativerepeater_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativerepeater_p_p.h @@ -53,9 +53,9 @@ // We mean it. // -#include "qdeclarativerepeater_p.h" +#include "private/qdeclarativerepeater_p.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativescalegrid.cpp b/src/declarative/graphicsitems/qdeclarativescalegrid.cpp index dbc7568..e68f645 100644 --- a/src/declarative/graphicsitems/qdeclarativescalegrid.cpp +++ b/src/declarative/graphicsitems/qdeclarativescalegrid.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativescalegrid_p_p.h" +#include "private/qdeclarativescalegrid_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativescalegrid_p_p.h b/src/declarative/graphicsitems/qdeclarativescalegrid_p_p.h index fbf9040..42813dd 100644 --- a/src/declarative/graphicsitems/qdeclarativescalegrid_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativescalegrid_p_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVESCALEGRID_H #define QDECLARATIVESCALEGRID_H -#include "qdeclarativeborderimage_p.h" +#include "private/qdeclarativeborderimage_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetext.cpp b/src/declarative/graphicsitems/qdeclarativetext.cpp index b0b5f6d..a4f3068 100644 --- a/src/declarative/graphicsitems/qdeclarativetext.cpp +++ b/src/declarative/graphicsitems/qdeclarativetext.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativetext_p.h" -#include "qdeclarativetext_p_p.h" +#include "private/qdeclarativetext_p.h" +#include "private/qdeclarativetext_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetext_p_p.h b/src/declarative/graphicsitems/qdeclarativetext_p_p.h index 0d9a0a6..85a65ce 100644 --- a/src/declarative/graphicsitems/qdeclarativetext_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativetext_p_p.h @@ -54,7 +54,7 @@ // #include "qdeclarativeitem.h" -#include "qdeclarativeitem_p.h" +#include "private/qdeclarativeitem_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativetextedit.cpp b/src/declarative/graphicsitems/qdeclarativetextedit.cpp index 03b2425..7374bc8 100644 --- a/src/declarative/graphicsitems/qdeclarativetextedit.cpp +++ b/src/declarative/graphicsitems/qdeclarativetextedit.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativetextedit_p.h" -#include "qdeclarativetextedit_p_p.h" +#include "private/qdeclarativetextedit_p.h" +#include "private/qdeclarativetextedit_p_p.h" -#include "qdeclarativeevents_p_p.h" +#include "private/qdeclarativeevents_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetextedit_p.h b/src/declarative/graphicsitems/qdeclarativetextedit_p.h index b1682c4..0e91e73 100644 --- a/src/declarative/graphicsitems/qdeclarativetextedit_p.h +++ b/src/declarative/graphicsitems/qdeclarativetextedit_p.h @@ -42,8 +42,8 @@ #ifndef QDECLARATIVETEXTEDIT_H #define QDECLARATIVETEXTEDIT_H -#include "qdeclarativetext_p.h" -#include "qdeclarativepainteditem_p.h" +#include "private/qdeclarativetext_p.h" +#include "private/qdeclarativepainteditem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetextedit_p_p.h b/src/declarative/graphicsitems/qdeclarativetextedit_p_p.h index dd2a29d..310db48 100644 --- a/src/declarative/graphicsitems/qdeclarativetextedit_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativetextedit_p_p.h @@ -54,7 +54,7 @@ // #include "qdeclarativeitem.h" -#include "qdeclarativepainteditem_p_p.h" +#include "private/qdeclarativepainteditem_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativetextinput.cpp b/src/declarative/graphicsitems/qdeclarativetextinput.cpp index 88801a4..29e43f9 100644 --- a/src/declarative/graphicsitems/qdeclarativetextinput.cpp +++ b/src/declarative/graphicsitems/qdeclarativetextinput.cpp @@ -39,8 +39,8 @@ ** ****************************************************************************/ -#include "qdeclarativetextinput_p.h" -#include "qdeclarativetextinput_p_p.h" +#include "private/qdeclarativetextinput_p.h" +#include "private/qdeclarativetextinput_p_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetextinput_p.h b/src/declarative/graphicsitems/qdeclarativetextinput_p.h index 6a61c2d..64aff7d 100644 --- a/src/declarative/graphicsitems/qdeclarativetextinput_p.h +++ b/src/declarative/graphicsitems/qdeclarativetextinput_p.h @@ -42,8 +42,8 @@ #ifndef QDECLARATIVETEXTINPUT_H #define QDECLARATIVETEXTINPUT_H -#include "qdeclarativetext_p.h" -#include "qdeclarativepainteditem_p.h" +#include "private/qdeclarativetext_p.h" +#include "private/qdeclarativepainteditem_p.h" #include #include diff --git a/src/declarative/graphicsitems/qdeclarativetextinput_p_p.h b/src/declarative/graphicsitems/qdeclarativetextinput_p_p.h index 5d17a55..9e44b15 100644 --- a/src/declarative/graphicsitems/qdeclarativetextinput_p_p.h +++ b/src/declarative/graphicsitems/qdeclarativetextinput_p_p.h @@ -42,9 +42,9 @@ #ifndef QDECLARATIVETEXTINPUT_P_H #define QDECLARATIVETEXTINPUT_P_H -#include "qdeclarativetextinput_p.h" +#include "private/qdeclarativetextinput_p.h" -#include "qdeclarativepainteditem_p_p.h" +#include "private/qdeclarativepainteditem_p_p.h" #include diff --git a/src/declarative/graphicsitems/qdeclarativetranslate.cpp b/src/declarative/graphicsitems/qdeclarativetranslate.cpp index 1c96fa4..17daac7 100644 --- a/src/declarative/graphicsitems/qdeclarativetranslate.cpp +++ b/src/declarative/graphicsitems/qdeclarativetranslate.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativetranslate_p.h" +#include "private/qdeclarativetranslate_p.h" #include #include #include diff --git a/src/declarative/graphicsitems/qdeclarativevisualitemmodel.cpp b/src/declarative/graphicsitems/qdeclarativevisualitemmodel.cpp index b31bbd0..2938f51 100644 --- a/src/declarative/graphicsitems/qdeclarativevisualitemmodel.cpp +++ b/src/declarative/graphicsitems/qdeclarativevisualitemmodel.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativevisualitemmodel_p.h" +#include "private/qdeclarativevisualitemmodel_p.h" #include "qdeclarativeitem.h" diff --git a/src/declarative/qml/parser/qdeclarativejs.g b/src/declarative/qml/parser/qdeclarativejs.g index 536989f..ba9338e 100644 --- a/src/declarative/qml/parser/qdeclarativejs.g +++ b/src/declarative/qml/parser/qdeclarativejs.g @@ -127,10 +127,10 @@ #include -#include "qdeclarativejsengine_p.h" -#include "qdeclarativejslexer_p.h" -#include "qdeclarativejsast_p.h" -#include "qdeclarativejsnodepool_p.h" +#include "private/qdeclarativejsengine_p.h" +#include "private/qdeclarativejslexer_p.h" +#include "private/qdeclarativejsast_p.h" +#include "private/qdeclarativejsnodepool_p.h" ./ @@ -195,10 +195,10 @@ #ifndef QDECLARATIVEJSPARSER_P_H #define QDECLARATIVEJSPARSER_P_H -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsgrammar_p.h" -#include "qdeclarativejsast_p.h" -#include "qdeclarativejsengine_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsgrammar_p.h" +#include "private/qdeclarativejsast_p.h" +#include "private/qdeclarativejsengine_p.h" #include #include @@ -375,7 +375,7 @@ protected: /. -#include "qdeclarativejsparser_p.h" +#include "private/qdeclarativejsparser_p.h" #include // diff --git a/src/declarative/qml/parser/qdeclarativejsast.cpp b/src/declarative/qml/parser/qdeclarativejsast.cpp index e3d3a66..000ef86 100644 --- a/src/declarative/qml/parser/qdeclarativejsast.cpp +++ b/src/declarative/qml/parser/qdeclarativejsast.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativejsast_p.h" +#include "private/qdeclarativejsast_p.h" -#include "qdeclarativejsastvisitor_p.h" +#include "private/qdeclarativejsastvisitor_p.h" QT_QML_BEGIN_NAMESPACE diff --git a/src/declarative/qml/parser/qdeclarativejsast_p.h b/src/declarative/qml/parser/qdeclarativejsast_p.h index b839413..0623a2a 100644 --- a/src/declarative/qml/parser/qdeclarativejsast_p.h +++ b/src/declarative/qml/parser/qdeclarativejsast_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativejsastvisitor_p.h" -#include "qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsastvisitor_p.h" +#include "private/qdeclarativejsglobal_p.h" #include diff --git a/src/declarative/qml/parser/qdeclarativejsastfwd_p.h b/src/declarative/qml/parser/qdeclarativejsastfwd_p.h index 5a444b7..d04aba5 100644 --- a/src/declarative/qml/parser/qdeclarativejsastfwd_p.h +++ b/src/declarative/qml/parser/qdeclarativejsastfwd_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEJSAST_FWD_P_H #define QDECLARATIVEJSAST_FWD_P_H -#include "qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsglobal_p.h" #include diff --git a/src/declarative/qml/parser/qdeclarativejsastvisitor.cpp b/src/declarative/qml/parser/qdeclarativejsastvisitor.cpp index d8002cf..d9a7be3 100644 --- a/src/declarative/qml/parser/qdeclarativejsastvisitor.cpp +++ b/src/declarative/qml/parser/qdeclarativejsastvisitor.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativejsastvisitor_p.h" +#include "private/qdeclarativejsastvisitor_p.h" QT_QML_BEGIN_NAMESPACE diff --git a/src/declarative/qml/parser/qdeclarativejsastvisitor_p.h b/src/declarative/qml/parser/qdeclarativejsastvisitor_p.h index 82abbcf..e1f45b4 100644 --- a/src/declarative/qml/parser/qdeclarativejsastvisitor_p.h +++ b/src/declarative/qml/parser/qdeclarativejsastvisitor_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativejsastfwd_p.h" -#include "qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsastfwd_p.h" +#include "private/qdeclarativejsglobal_p.h" QT_QML_BEGIN_NAMESPACE diff --git a/src/declarative/qml/parser/qdeclarativejsengine_p.cpp b/src/declarative/qml/parser/qdeclarativejsengine_p.cpp index 7d39134..0527b3c 100644 --- a/src/declarative/qml/parser/qdeclarativejsengine_p.cpp +++ b/src/declarative/qml/parser/qdeclarativejsengine_p.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativejsengine_p.h" +#include "private/qdeclarativejsengine_p.h" -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsnodepool_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsnodepool_p.h" #include #include diff --git a/src/declarative/qml/parser/qdeclarativejsengine_p.h b/src/declarative/qml/parser/qdeclarativejsengine_p.h index 61188f7..1389244 100644 --- a/src/declarative/qml/parser/qdeclarativejsengine_p.h +++ b/src/declarative/qml/parser/qdeclarativejsengine_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsastfwd_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsastfwd_p.h" #include #include diff --git a/src/declarative/qml/parser/qdeclarativejsgrammar.cpp b/src/declarative/qml/parser/qdeclarativejsgrammar.cpp index 39a2d3f..52e979a 100644 --- a/src/declarative/qml/parser/qdeclarativejsgrammar.cpp +++ b/src/declarative/qml/parser/qdeclarativejsgrammar.cpp @@ -40,7 +40,7 @@ ****************************************************************************/ // This file was generated by qlalr - DO NOT EDIT! -#include "qdeclarativejsgrammar_p.h" +#include "private/qdeclarativejsgrammar_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/parser/qdeclarativejslexer.cpp b/src/declarative/qml/parser/qdeclarativejslexer.cpp index 6404be3..a616146 100644 --- a/src/declarative/qml/parser/qdeclarativejslexer.cpp +++ b/src/declarative/qml/parser/qdeclarativejslexer.cpp @@ -43,11 +43,11 @@ #include #endif -#include "qdeclarativejslexer_p.h" +#include "private/qdeclarativejslexer_p.h" -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsengine_p.h" -#include "qdeclarativejsgrammar_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsengine_p.h" +#include "private/qdeclarativejsgrammar_p.h" #include diff --git a/src/declarative/qml/parser/qdeclarativejslexer_p.h b/src/declarative/qml/parser/qdeclarativejslexer_p.h index 71bd08c..f2a2fa5 100644 --- a/src/declarative/qml/parser/qdeclarativejslexer_p.h +++ b/src/declarative/qml/parser/qdeclarativejslexer_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsglobal_p.h" #include diff --git a/src/declarative/qml/parser/qdeclarativejsmemorypool_p.h b/src/declarative/qml/parser/qdeclarativejsmemorypool_p.h index c8d52d0..51e6e54 100644 --- a/src/declarative/qml/parser/qdeclarativejsmemorypool_p.h +++ b/src/declarative/qml/parser/qdeclarativejsmemorypool_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsglobal_p.h" #include #include diff --git a/src/declarative/qml/parser/qdeclarativejsnodepool_p.h b/src/declarative/qml/parser/qdeclarativejsnodepool_p.h index 94637fd..ffa9ee5 100644 --- a/src/declarative/qml/parser/qdeclarativejsnodepool_p.h +++ b/src/declarative/qml/parser/qdeclarativejsnodepool_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsmemorypool_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsmemorypool_p.h" #include #include diff --git a/src/declarative/qml/parser/qdeclarativejsparser.cpp b/src/declarative/qml/parser/qdeclarativejsparser.cpp index 170c7fa..8afb93d 100644 --- a/src/declarative/qml/parser/qdeclarativejsparser.cpp +++ b/src/declarative/qml/parser/qdeclarativejsparser.cpp @@ -44,14 +44,14 @@ #include -#include "qdeclarativejsengine_p.h" -#include "qdeclarativejslexer_p.h" -#include "qdeclarativejsast_p.h" -#include "qdeclarativejsnodepool_p.h" +#include "private/qdeclarativejsengine_p.h" +#include "private/qdeclarativejslexer_p.h" +#include "private/qdeclarativejsast_p.h" +#include "private/qdeclarativejsnodepool_p.h" -#include "qdeclarativejsparser_p.h" +#include "private/qdeclarativejsparser_p.h" #include // diff --git a/src/declarative/qml/parser/qdeclarativejsparser_p.h b/src/declarative/qml/parser/qdeclarativejsparser_p.h index 8838fbe..005bd3f 100644 --- a/src/declarative/qml/parser/qdeclarativejsparser_p.h +++ b/src/declarative/qml/parser/qdeclarativejsparser_p.h @@ -59,10 +59,10 @@ #ifndef QDECLARATIVEJSPARSER_P_H #define QDECLARATIVEJSPARSER_P_H -#include "qdeclarativejsglobal_p.h" -#include "qdeclarativejsgrammar_p.h" -#include "qdeclarativejsast_p.h" -#include "qdeclarativejsengine_p.h" +#include "private/qdeclarativejsglobal_p.h" +#include "private/qdeclarativejsgrammar_p.h" +#include "private/qdeclarativejsast_p.h" +#include "private/qdeclarativejsengine_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativebinding.cpp b/src/declarative/qml/qdeclarativebinding.cpp index a7047ab..71cf3cb 100644 --- a/src/declarative/qml/qdeclarativebinding.cpp +++ b/src/declarative/qml/qdeclarativebinding.cpp @@ -39,15 +39,15 @@ ** ****************************************************************************/ -#include "qdeclarativebinding_p.h" -#include "qdeclarativebinding_p_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativebinding_p_p.h" #include "qdeclarative.h" #include "qdeclarativecontext.h" #include "qdeclarativeinfo.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativedeclarativedata_p.h" -#include "qdeclarativestringconverters_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativestringconverters_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativebinding_p.h b/src/declarative/qml/qdeclarativebinding_p.h index 56f1715..2789738 100644 --- a/src/declarative/qml/qdeclarativebinding_p.h +++ b/src/declarative/qml/qdeclarativebinding_p.h @@ -57,7 +57,7 @@ #include "qdeclarativepropertyvaluesource.h" #include "qdeclarativeexpression.h" #include "qdeclarativeproperty.h" -#include "qdeclarativeproperty_p.h" +#include "private/qdeclarativeproperty_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativebinding_p_p.h b/src/declarative/qml/qdeclarativebinding_p_p.h index 041103c..617ec4b 100644 --- a/src/declarative/qml/qdeclarativebinding_p_p.h +++ b/src/declarative/qml/qdeclarativebinding_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativebinding_p.h" +#include "private/qdeclarativebinding_p.h" #include "qdeclarativeproperty.h" -#include "qdeclarativeexpression_p.h" +#include "private/qdeclarativeexpression_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativeboundsignal.cpp b/src/declarative/qml/qdeclarativeboundsignal.cpp index 6a5a102..762c6428 100644 --- a/src/declarative/qml/qdeclarativeboundsignal.cpp +++ b/src/declarative/qml/qdeclarativeboundsignal.cpp @@ -39,16 +39,16 @@ ** ****************************************************************************/ -#include "qdeclarativeboundsignal_p.h" +#include "private/qdeclarativeboundsignal_p.h" -#include "qmetaobjectbuilder_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeexpression_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativemetatype_p.h" +#include "private/qmetaobjectbuilder_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeexpression_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativemetatype_p.h" #include "qdeclarative.h" #include "qdeclarativecontext.h" -#include "qdeclarativeglobal_p.h" +#include "private/qdeclarativeglobal_p.h" #include diff --git a/src/declarative/qml/qdeclarativeclassfactory.cpp b/src/declarative/qml/qdeclarativeclassfactory.cpp index ae4975c..798eacd 100644 --- a/src/declarative/qml/qdeclarativeclassfactory.cpp +++ b/src/declarative/qml/qdeclarativeclassfactory.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeclassfactory_p.h" +#include "private/qdeclarativeclassfactory_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativecleanup.cpp b/src/declarative/qml/qdeclarativecleanup.cpp index 0fa49d1..4f48ba1 100644 --- a/src/declarative/qml/qdeclarativecleanup.cpp +++ b/src/declarative/qml/qdeclarativecleanup.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativecleanup_p.h" +#include "private/qdeclarativecleanup_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativecompiledbindings.cpp b/src/declarative/qml/qdeclarativecompiledbindings.cpp index 67750a4..53143d5 100644 --- a/src/declarative/qml/qdeclarativecompiledbindings.cpp +++ b/src/declarative/qml/qdeclarativecompiledbindings.cpp @@ -41,7 +41,7 @@ // #define COMPILEDBINDINGS_DEBUG -#include "qdeclarativecompiledbindings_p.h" +#include "private/qdeclarativecompiledbindings_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecompiledbindings_p.h b/src/declarative/qml/qdeclarativecompiledbindings_p.h index 8776c08..a17bc84 100644 --- a/src/declarative/qml/qdeclarativecompiledbindings_p.h +++ b/src/declarative/qml/qdeclarativecompiledbindings_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativeexpression_p.h" -#include "qdeclarativebinding_p.h" +#include "private/qdeclarativeexpression_p.h" +#include "private/qdeclarativebinding_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/qml/qdeclarativecompileddata.cpp b/src/declarative/qml/qdeclarativecompileddata.cpp index dfbf453..5d73d89 100644 --- a/src/declarative/qml/qdeclarativecompileddata.cpp +++ b/src/declarative/qml/qdeclarativecompileddata.cpp @@ -39,12 +39,12 @@ ** ****************************************************************************/ -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativecompiler_p.h" #include "qdeclarativeengine.h" #include "qdeclarativecomponent.h" -#include "qdeclarativecomponent_p.h" +#include "private/qdeclarativecomponent_p.h" #include "qdeclarativecontext.h" -#include "qdeclarativecontext_p.h" +#include "private/qdeclarativecontext_p.h" #include diff --git a/src/declarative/qml/qdeclarativecompiler.cpp b/src/declarative/qml/qdeclarativecompiler.cpp index 56af5f5..f4ccce1 100644 --- a/src/declarative/qml/qdeclarativecompiler.cpp +++ b/src/declarative/qml/qdeclarativecompiler.cpp @@ -39,33 +39,33 @@ ** ****************************************************************************/ -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativecompiler_p.h" -#include "qdeclarativecompositetypedata_p.h" -#include "qdeclarativeparser_p.h" -#include "qdeclarativescriptparser_p.h" +#include "private/qdeclarativecompositetypedata_p.h" +#include "private/qdeclarativeparser_p.h" +#include "private/qdeclarativescriptparser_p.h" #include "qdeclarativepropertyvaluesource.h" #include "qdeclarativecomponent.h" -#include "qmetaobjectbuilder_p.h" -#include "qdeclarativestringconverters_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qmetaobjectbuilder_p.h" +#include "private/qdeclarativestringconverters_p.h" +#include "private/qdeclarativeengine_p.h" #include "qdeclarativeengine.h" #include "qdeclarativecontext.h" -#include "qdeclarativemetatype_p.h" -#include "qdeclarativecustomparser_p_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativecomponent_p.h" +#include "private/qdeclarativemetatype_p.h" +#include "private/qdeclarativecustomparser_p_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativecomponent_p.h" #include "parser/qdeclarativejsast_p.h" -#include "qdeclarativevmemetaobject_p.h" -#include "qdeclarativeexpression_p.h" -#include "qdeclarativeproperty_p.h" -#include "qdeclarativerewrite_p.h" +#include "private/qdeclarativevmemetaobject_p.h" +#include "private/qdeclarativeexpression_p.h" +#include "private/qdeclarativeproperty_p.h" +#include "private/qdeclarativerewrite_p.h" #include "qdeclarativescriptstring.h" -#include "qdeclarativeglobal_p.h" -#include "qdeclarativescriptparser_p.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativecompiledbindings_p.h" -#include "qdeclarativeglobalscriptclass_p.h" +#include "private/qdeclarativeglobal_p.h" +#include "private/qdeclarativescriptparser_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativecompiledbindings_p.h" +#include "private/qdeclarativeglobalscriptclass_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecompiler_p.h b/src/declarative/qml/qdeclarativecompiler_p.h index e5836d1..11364bb 100644 --- a/src/declarative/qml/qdeclarativecompiler_p.h +++ b/src/declarative/qml/qdeclarativecompiler_p.h @@ -55,14 +55,14 @@ #include "qdeclarative.h" #include "qdeclarativeerror.h" -#include "qdeclarativeinstruction_p.h" -#include "qdeclarativecompositetypemanager_p.h" -#include "qdeclarativeparser_p.h" -#include "qdeclarativeengine_p.h" -#include "qbitfield_p.h" -#include "qdeclarativepropertycache_p.h" -#include "qdeclarativeintegercache_p.h" -#include "qdeclarativetypenamecache_p.h" +#include "private/qdeclarativeinstruction_p.h" +#include "private/qdeclarativecompositetypemanager_p.h" +#include "private/qdeclarativeparser_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qbitfield_p.h" +#include "private/qdeclarativepropertycache_p.h" +#include "private/qdeclarativeintegercache_p.h" +#include "private/qdeclarativetypenamecache_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecomponent.cpp b/src/declarative/qml/qdeclarativecomponent.cpp index ec23458..e180374 100644 --- a/src/declarative/qml/qdeclarativecomponent.cpp +++ b/src/declarative/qml/qdeclarativecomponent.cpp @@ -40,19 +40,19 @@ ****************************************************************************/ #include "qdeclarativecomponent.h" -#include "qdeclarativecomponent_p.h" +#include "private/qdeclarativecomponent_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativecompositetypedata_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativevme_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativecompositetypedata_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativevme_p.h" #include "qdeclarative.h" #include "qdeclarativeengine.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativebinding_p_p.h" -#include "qdeclarativeglobal_p.h" -#include "qdeclarativescriptparser_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativebinding_p_p.h" +#include "private/qdeclarativeglobal_p.h" +#include "private/qdeclarativescriptparser_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecomponent_p.h b/src/declarative/qml/qdeclarativecomponent_p.h index b44aeef..dfe327b 100644 --- a/src/declarative/qml/qdeclarativecomponent_p.h +++ b/src/declarative/qml/qdeclarativecomponent_p.h @@ -55,9 +55,9 @@ #include "qdeclarativecomponent.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativecompositetypemanager_p.h" -#include "qbitfield_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativecompositetypemanager_p.h" +#include "private/qbitfield_p.h" #include "qdeclarativeerror.h" #include "qdeclarative.h" diff --git a/src/declarative/qml/qdeclarativecompositetypedata_p.h b/src/declarative/qml/qdeclarativecompositetypedata_p.h index 04d0c63..47cb3b3 100644 --- a/src/declarative/qml/qdeclarativecompositetypedata_p.h +++ b/src/declarative/qml/qdeclarativecompositetypedata_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" #include diff --git a/src/declarative/qml/qdeclarativecompositetypemanager.cpp b/src/declarative/qml/qdeclarativecompositetypemanager.cpp index eb8c9eb..55723ea 100644 --- a/src/declarative/qml/qdeclarativecompositetypemanager.cpp +++ b/src/declarative/qml/qdeclarativecompositetypemanager.cpp @@ -39,15 +39,15 @@ ** ****************************************************************************/ -#include "qdeclarativecompositetypemanager_p.h" +#include "private/qdeclarativecompositetypemanager_p.h" -#include "qdeclarativecompositetypedata_p.h" -#include "qdeclarativescriptparser_p.h" +#include "private/qdeclarativecompositetypedata_p.h" +#include "private/qdeclarativescriptparser_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" #include "qdeclarativecomponent.h" -#include "qdeclarativecomponent_p.h" -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativecomponent_p.h" +#include "private/qdeclarativecompiler_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecompositetypemanager_p.h b/src/declarative/qml/qdeclarativecompositetypemanager_p.h index a572e0c..5c82c4c 100644 --- a/src/declarative/qml/qdeclarativecompositetypemanager_p.h +++ b/src/declarative/qml/qdeclarativecompositetypemanager_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativescriptparser_p.h" -#include "qdeclarativerefcount_p.h" +#include "private/qdeclarativescriptparser_p.h" +#include "private/qdeclarativerefcount_p.h" #include "qdeclarativeerror.h" #include "qdeclarativeengine.h" diff --git a/src/declarative/qml/qdeclarativecontext.cpp b/src/declarative/qml/qdeclarativecontext.cpp index a9224ad..55c2f7c 100644 --- a/src/declarative/qml/qdeclarativecontext.cpp +++ b/src/declarative/qml/qdeclarativecontext.cpp @@ -40,14 +40,14 @@ ****************************************************************************/ #include "qdeclarativecontext.h" -#include "qdeclarativecontext_p.h" +#include "private/qdeclarativecontext_p.h" -#include "qdeclarativeexpression_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeexpression_p.h" +#include "private/qdeclarativeengine_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativecompiledbindings_p.h" +#include "private/qdeclarativecompiledbindings_p.h" #include "qdeclarativeinfo.h" -#include "qdeclarativeglobalscriptclass_p.h" +#include "private/qdeclarativeglobalscriptclass_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecontext_p.h b/src/declarative/qml/qdeclarativecontext_p.h index 397f37a..6c5a1f7 100644 --- a/src/declarative/qml/qdeclarativecontext_p.h +++ b/src/declarative/qml/qdeclarativecontext_p.h @@ -55,19 +55,19 @@ #include "qdeclarativecontext.h" -#include "qdeclarativedeclarativedata_p.h" -#include "qdeclarativeintegercache_p.h" -#include "qdeclarativetypenamecache_p.h" -#include "qdeclarativenotifier_p.h" +#include "private/qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativeintegercache_p.h" +#include "private/qdeclarativetypenamecache_p.h" +#include "private/qdeclarativenotifier_p.h" #include "qdeclarativelist.h" -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include #include #include #include -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativecontextscriptclass.cpp b/src/declarative/qml/qdeclarativecontextscriptclass.cpp index 2559224..8566744 100644 --- a/src/declarative/qml/qdeclarativecontextscriptclass.cpp +++ b/src/declarative/qml/qdeclarativecontextscriptclass.cpp @@ -39,13 +39,13 @@ ** ****************************************************************************/ -#include "qdeclarativecontextscriptclass_p.h" +#include "private/qdeclarativecontextscriptclass_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativetypenamescriptclass_p.h" -#include "qdeclarativelistscriptclass_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativetypenamescriptclass_p.h" +#include "private/qdeclarativelistscriptclass_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativecontextscriptclass_p.h b/src/declarative/qml/qdeclarativecontextscriptclass_p.h index 93e4b20..b89f0cd 100644 --- a/src/declarative/qml/qdeclarativecontextscriptclass_p.h +++ b/src/declarative/qml/qdeclarativecontextscriptclass_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativetypenamecache_p.h" -#include "qdeclarativescriptclass_p.h" +#include "private/qdeclarativetypenamecache_p.h" +#include "private/qdeclarativescriptclass_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativecustomparser.cpp b/src/declarative/qml/qdeclarativecustomparser.cpp index a3a511c..8b6ee7c 100644 --- a/src/declarative/qml/qdeclarativecustomparser.cpp +++ b/src/declarative/qml/qdeclarativecustomparser.cpp @@ -39,11 +39,11 @@ ** ****************************************************************************/ -#include "qdeclarativecustomparser_p.h" -#include "qdeclarativecustomparser_p_p.h" +#include "private/qdeclarativecustomparser_p.h" +#include "private/qdeclarativecustomparser_p_p.h" -#include "qdeclarativeparser_p.h" -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativeparser_p.h" +#include "private/qdeclarativecompiler_p.h" #include diff --git a/src/declarative/qml/qdeclarativecustomparser_p.h b/src/declarative/qml/qdeclarativecustomparser_p.h index f9bf513..da0358a 100644 --- a/src/declarative/qml/qdeclarativecustomparser_p.h +++ b/src/declarative/qml/qdeclarativecustomparser_p.h @@ -53,9 +53,9 @@ // We mean it. // -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativemetatype_p.h" #include "qdeclarativeerror.h" -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativecustomparser_p_p.h b/src/declarative/qml/qdeclarativecustomparser_p_p.h index b580db6..a7e213c 100644 --- a/src/declarative/qml/qdeclarativecustomparser_p_p.h +++ b/src/declarative/qml/qdeclarativecustomparser_p_p.h @@ -53,9 +53,9 @@ // We mean it. // -#include "qdeclarativecustomparser_p.h" +#include "private/qdeclarativecustomparser_p.h" -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include diff --git a/src/declarative/qml/qdeclarativedeclarativedata_p.h b/src/declarative/qml/qdeclarativedeclarativedata_p.h index d1d063a..adfff19 100644 --- a/src/declarative/qml/qdeclarativedeclarativedata_p.h +++ b/src/declarative/qml/qdeclarativedeclarativedata_p.h @@ -55,7 +55,7 @@ #include #include -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativedirparser.cpp b/src/declarative/qml/qdeclarativedirparser.cpp index 0e82098..1e3b37b 100644 --- a/src/declarative/qml/qdeclarativedirparser.cpp +++ b/src/declarative/qml/qdeclarativedirparser.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativedirparser_p.h" +#include "private/qdeclarativedirparser_p.h" #include "qdeclarativeerror.h" #include diff --git a/src/declarative/qml/qdeclarativedom.cpp b/src/declarative/qml/qdeclarativedom.cpp index dec30a1..a3cddb5 100644 --- a/src/declarative/qml/qdeclarativedom.cpp +++ b/src/declarative/qml/qdeclarativedom.cpp @@ -39,14 +39,14 @@ ** ****************************************************************************/ -#include "qdeclarativedom_p.h" -#include "qdeclarativedom_p_p.h" - -#include "qdeclarativecompositetypedata_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativescriptparser_p.h" -#include "qdeclarativeglobal_p.h" +#include "private/qdeclarativedom_p.h" +#include "private/qdeclarativedom_p_p.h" + +#include "private/qdeclarativecompositetypedata_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativescriptparser_p.h" +#include "private/qdeclarativeglobal_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativedom_p_p.h b/src/declarative/qml/qdeclarativedom_p_p.h index a065282..7ce99ec 100644 --- a/src/declarative/qml/qdeclarativedom_p_p.h +++ b/src/declarative/qml/qdeclarativedom_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include diff --git a/src/declarative/qml/qdeclarativeengine.cpp b/src/declarative/qml/qdeclarativeengine.cpp index 9442705..52a1f45 100644 --- a/src/declarative/qml/qdeclarativeengine.cpp +++ b/src/declarative/qml/qdeclarativeengine.cpp @@ -39,35 +39,35 @@ ** ****************************************************************************/ -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativeglobalscriptclass_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativeglobalscriptclass_p.h" #include "qdeclarative.h" #include "qdeclarativecontext.h" #include "qdeclarativeexpression.h" #include "qdeclarativecomponent.h" -#include "qdeclarativebinding_p_p.h" -#include "qdeclarativevme_p.h" -#include "qdeclarativeenginedebug_p.h" -#include "qdeclarativestringconverters_p.h" -#include "qdeclarativexmlhttprequest_p.h" -#include "qdeclarativesqldatabase_p.h" -#include "qdeclarativetypenamescriptclass_p.h" -#include "qdeclarativelistscriptclass_p.h" +#include "private/qdeclarativebinding_p_p.h" +#include "private/qdeclarativevme_p.h" +#include "private/qdeclarativeenginedebug_p.h" +#include "private/qdeclarativestringconverters_p.h" +#include "private/qdeclarativexmlhttprequest_p.h" +#include "private/qdeclarativesqldatabase_p.h" +#include "private/qdeclarativetypenamescriptclass_p.h" +#include "private/qdeclarativelistscriptclass_p.h" #include "qdeclarativescriptstring.h" -#include "qdeclarativeglobal_p.h" -#include "qdeclarativeworkerscript_p.h" -#include "qdeclarativecomponent_p.h" -#include "qdeclarativescriptclass_p.h" +#include "private/qdeclarativeglobal_p.h" +#include "private/qdeclarativeworkerscript_p.h" +#include "private/qdeclarativecomponent_p.h" +#include "private/qdeclarativescriptclass_p.h" #include "qdeclarativenetworkaccessmanagerfactory.h" #include "qdeclarativeimageprovider.h" -#include "qdeclarativedirparser_p.h" +#include "private/qdeclarativedirparser_p.h" #include "qdeclarativeextensioninterface.h" -#include "qdeclarativelist_p.h" -#include "qdeclarativetypenamecache_p.h" +#include "private/qdeclarativelist_p.h" +#include "private/qdeclarativetypenamecache_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeengine_p.h b/src/declarative/qml/qdeclarativeengine_p.h index 5aff30d..45089d0 100644 --- a/src/declarative/qml/qdeclarativeengine_p.h +++ b/src/declarative/qml/qdeclarativeengine_p.h @@ -55,21 +55,21 @@ #include "qdeclarativeengine.h" -#include "qdeclarativeclassfactory_p.h" -#include "qdeclarativecompositetypemanager_p.h" -#include "qpodvector_p.h" +#include "private/qdeclarativeclassfactory_p.h" +#include "private/qdeclarativecompositetypemanager_p.h" +#include "private/qpodvector_p.h" #include "qdeclarative.h" -#include "qdeclarativevaluetype_p.h" +#include "private/qdeclarativevaluetype_p.h" #include "qdeclarativecontext.h" -#include "qdeclarativecontext_p.h" +#include "private/qdeclarativecontext_p.h" #include "qdeclarativeexpression.h" -#include "qdeclarativeproperty_p.h" -#include "qdeclarativepropertycache_p.h" -#include "qdeclarativeobjectscriptclass_p.h" -#include "qdeclarativecontextscriptclass_p.h" -#include "qdeclarativevaluetypescriptclass_p.h" -#include "qdeclarativemetatype_p.h" -#include "qdeclarativedirparser_p.h" +#include "private/qdeclarativeproperty_p.h" +#include "private/qdeclarativepropertycache_p.h" +#include "private/qdeclarativeobjectscriptclass_p.h" +#include "private/qdeclarativecontextscriptclass_p.h" +#include "private/qdeclarativevaluetypescriptclass_p.h" +#include "private/qdeclarativemetatype_p.h" +#include "private/qdeclarativedirparser_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeenginedebug.cpp b/src/declarative/qml/qdeclarativeenginedebug.cpp index d30aa8e..3455883 100644 --- a/src/declarative/qml/qdeclarativeenginedebug.cpp +++ b/src/declarative/qml/qdeclarativeenginedebug.cpp @@ -39,17 +39,17 @@ ** ****************************************************************************/ -#include "qdeclarativeenginedebug_p.h" +#include "private/qdeclarativeenginedebug_p.h" -#include "qdeclarativeboundsignal_p.h" +#include "private/qdeclarativeboundsignal_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativemetatype_p.h" #include "qdeclarativeproperty.h" -#include "qdeclarativeproperty_p.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativewatcher_p.h" -#include "qdeclarativevaluetype_p.h" +#include "private/qdeclarativeproperty_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativewatcher_p.h" +#include "private/qdeclarativevaluetype_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeexpression.cpp b/src/declarative/qml/qdeclarativeexpression.cpp index b7ee3a4..a250f21 100644 --- a/src/declarative/qml/qdeclarativeexpression.cpp +++ b/src/declarative/qml/qdeclarativeexpression.cpp @@ -40,13 +40,13 @@ ****************************************************************************/ #include "qdeclarativeexpression.h" -#include "qdeclarativeexpression_p.h" +#include "private/qdeclarativeexpression_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativerewrite_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativeglobalscriptclass_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativerewrite_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativeglobalscriptclass_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeexpression_p.h b/src/declarative/qml/qdeclarativeexpression_p.h index 5adaa89..9a90fb6 100644 --- a/src/declarative/qml/qdeclarativeexpression_p.h +++ b/src/declarative/qml/qdeclarativeexpression_p.h @@ -55,8 +55,8 @@ #include "qdeclarativeexpression.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeguard_p.h" #include diff --git a/src/declarative/qml/qdeclarativefastproperties.cpp b/src/declarative/qml/qdeclarativefastproperties.cpp index 088d329..eb69b6a 100644 --- a/src/declarative/qml/qdeclarativefastproperties.cpp +++ b/src/declarative/qml/qdeclarativefastproperties.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativefastproperties_p.h" +#include "private/qdeclarativefastproperties_p.h" #include diff --git a/src/declarative/qml/qdeclarativeglobalscriptclass.cpp b/src/declarative/qml/qdeclarativeglobalscriptclass.cpp index 9ee2fe5..fc802b4 100644 --- a/src/declarative/qml/qdeclarativeglobalscriptclass.cpp +++ b/src/declarative/qml/qdeclarativeglobalscriptclass.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeglobalscriptclass_p.h" +#include "private/qdeclarativeglobalscriptclass_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeguard_p.h b/src/declarative/qml/qdeclarativeguard_p.h index 0861e9a..747d58f 100644 --- a/src/declarative/qml/qdeclarativeguard_p.h +++ b/src/declarative/qml/qdeclarativeguard_p.h @@ -99,7 +99,7 @@ QT_END_NAMESPACE Q_DECLARE_METATYPE(QDeclarativeGuard); -#include "qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativedeclarativedata_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativeinfo.cpp b/src/declarative/qml/qdeclarativeinfo.cpp index 5146bb6..6199aa9 100644 --- a/src/declarative/qml/qdeclarativeinfo.cpp +++ b/src/declarative/qml/qdeclarativeinfo.cpp @@ -41,10 +41,10 @@ #include "qdeclarativeinfo.h" -#include "qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativedeclarativedata_p.h" #include "qdeclarativecontext.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativemetatype_p.h" #include diff --git a/src/declarative/qml/qdeclarativeinstruction.cpp b/src/declarative/qml/qdeclarativeinstruction.cpp index 9083ab3..3cf4d2b 100644 --- a/src/declarative/qml/qdeclarativeinstruction.cpp +++ b/src/declarative/qml/qdeclarativeinstruction.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativeinstruction_p.h" +#include "private/qdeclarativeinstruction_p.h" -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativecompiler_p.h" #include diff --git a/src/declarative/qml/qdeclarativeintegercache.cpp b/src/declarative/qml/qdeclarativeintegercache.cpp index be36471..d1e17a0 100644 --- a/src/declarative/qml/qdeclarativeintegercache.cpp +++ b/src/declarative/qml/qdeclarativeintegercache.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativeintegercache_p.h" +#include "private/qdeclarativeintegercache_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativemetatype_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativeintegercache_p.h b/src/declarative/qml/qdeclarativeintegercache_p.h index 5fb5a76..b3a3796 100644 --- a/src/declarative/qml/qdeclarativeintegercache_p.h +++ b/src/declarative/qml/qdeclarativeintegercache_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativerefcount_p.h" -#include "qdeclarativecleanup_p.h" +#include "private/qdeclarativerefcount_p.h" +#include "private/qdeclarativecleanup_p.h" #include diff --git a/src/declarative/qml/qdeclarativelist.cpp b/src/declarative/qml/qdeclarativelist.cpp index 9691f32..45b8cd7 100644 --- a/src/declarative/qml/qdeclarativelist.cpp +++ b/src/declarative/qml/qdeclarativelist.cpp @@ -40,9 +40,9 @@ ****************************************************************************/ #include "qdeclarativelist.h" -#include "qdeclarativelist_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeproperty_p.h" +#include "private/qdeclarativelist_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeproperty_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativelist.h b/src/declarative/qml/qdeclarativelist.h index 219aa4b..399b2a1 100644 --- a/src/declarative/qml/qdeclarativelist.h +++ b/src/declarative/qml/qdeclarativelist.h @@ -145,7 +145,7 @@ private: QT_END_NAMESPACE -Q_DECLARE_METATYPE(QDeclarativeListReference); +Q_DECLARE_METATYPE(QDeclarativeListReference) QT_END_HEADER diff --git a/src/declarative/qml/qdeclarativelist_p.h b/src/declarative/qml/qdeclarativelist_p.h index 45a805a..b17f84a 100644 --- a/src/declarative/qml/qdeclarativelist_p.h +++ b/src/declarative/qml/qdeclarativelist_p.h @@ -54,7 +54,7 @@ // #include "qdeclarativelist.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativelistscriptclass.cpp b/src/declarative/qml/qdeclarativelistscriptclass.cpp index ea04ad9..3958dd5 100644 --- a/src/declarative/qml/qdeclarativelistscriptclass.cpp +++ b/src/declarative/qml/qdeclarativelistscriptclass.cpp @@ -39,11 +39,11 @@ ** ****************************************************************************/ -#include "qdeclarativelistscriptclass_p.h" +#include "private/qdeclarativelistscriptclass_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeguard_p.h" -#include "qdeclarativelist_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeguard_p.h" +#include "private/qdeclarativelist_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativemetatype.cpp b/src/declarative/qml/qdeclarativemetatype.cpp index c512d97..af21765 100644 --- a/src/declarative/qml/qdeclarativemetatype.cpp +++ b/src/declarative/qml/qdeclarativemetatype.cpp @@ -39,11 +39,11 @@ ** ****************************************************************************/ -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativemetatype_p.h" -#include "qdeclarativeproxymetaobject_p.h" -#include "qdeclarativecustomparser_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeproxymetaobject_p.h" +#include "private/qdeclarativecustomparser_p.h" +#include "private/qdeclarativeguard_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativenotifier.cpp b/src/declarative/qml/qdeclarativenotifier.cpp index b12bf77..8c946f8 100644 --- a/src/declarative/qml/qdeclarativenotifier.cpp +++ b/src/declarative/qml/qdeclarativenotifier.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativenotifier_p.h" +#include "private/qdeclarativenotifier_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativenotifier_p.h b/src/declarative/qml/qdeclarativenotifier_p.h index 2a9660d..0781caf 100644 --- a/src/declarative/qml/qdeclarativenotifier_p.h +++ b/src/declarative/qml/qdeclarativenotifier_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVENOTIFIER_P_H #define QDECLARATIVENOTIFIER_P_H -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativeobjectscriptclass.cpp b/src/declarative/qml/qdeclarativeobjectscriptclass.cpp index 84c9bef..d3ee67b 100644 --- a/src/declarative/qml/qdeclarativeobjectscriptclass.cpp +++ b/src/declarative/qml/qdeclarativeobjectscriptclass.cpp @@ -39,16 +39,16 @@ ** ****************************************************************************/ -#include "qdeclarativeobjectscriptclass_p.h" - -#include "qdeclarativeengine_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativedeclarativedata_p.h" -#include "qdeclarativetypenamescriptclass_p.h" -#include "qdeclarativelistscriptclass_p.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativeguard_p.h" -#include "qdeclarativevmemetaobject_p.h" +#include "private/qdeclarativeobjectscriptclass_p.h" + +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativetypenamescriptclass_p.h" +#include "private/qdeclarativelistscriptclass_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativeguard_p.h" +#include "private/qdeclarativevmemetaobject_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeobjectscriptclass_p.h b/src/declarative/qml/qdeclarativeobjectscriptclass_p.h index 396b782..8941ae0 100644 --- a/src/declarative/qml/qdeclarativeobjectscriptclass_p.h +++ b/src/declarative/qml/qdeclarativeobjectscriptclass_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativepropertycache_p.h" -#include "qdeclarativetypenamecache_p.h" +#include "private/qdeclarativepropertycache_p.h" +#include "private/qdeclarativetypenamecache_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeparser.cpp b/src/declarative/qml/qdeclarativeparser.cpp index 6e6080e..69186b6 100644 --- a/src/declarative/qml/qdeclarativeparser.cpp +++ b/src/declarative/qml/qdeclarativeparser.cpp @@ -39,16 +39,16 @@ ** ****************************************************************************/ -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include "qdeclarativepropertyvaluesource.h" -#include "qdeclarativevme_p.h" +#include "private/qdeclarativevme_p.h" #include "qdeclarative.h" -#include "qdeclarativecomponent_p.h" +#include "private/qdeclarativecomponent_p.h" #include "qdeclarativecomponent.h" -#include "qmetaobjectbuilder_p.h" -#include "qdeclarativevmemetaobject_p.h" -#include "qdeclarativecompiler_p.h" +#include "private/qmetaobjectbuilder_p.h" +#include "private/qdeclarativevmemetaobject_p.h" +#include "private/qdeclarativecompiler_p.h" #include "parser/qdeclarativejsast_p.h" #include "parser/qdeclarativejsengine_p.h" diff --git a/src/declarative/qml/qdeclarativeparser_p.h b/src/declarative/qml/qdeclarativeparser_p.h index f432024..fd3e22f 100644 --- a/src/declarative/qml/qdeclarativeparser_p.h +++ b/src/declarative/qml/qdeclarativeparser_p.h @@ -54,7 +54,7 @@ // #include "qdeclarative.h" -#include "qdeclarativerefcount_p.h" +#include "private/qdeclarativerefcount_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeproperty.cpp b/src/declarative/qml/qdeclarativeproperty.cpp index caa1acf..affb6b9 100644 --- a/src/declarative/qml/qdeclarativeproperty.cpp +++ b/src/declarative/qml/qdeclarativeproperty.cpp @@ -40,20 +40,20 @@ ****************************************************************************/ #include "qdeclarativeproperty.h" -#include "qdeclarativeproperty_p.h" +#include "private/qdeclarativeproperty_p.h" -#include "qdeclarativecompositetypedata_p.h" +#include "private/qdeclarativecompositetypedata_p.h" #include "qdeclarative.h" -#include "qdeclarativebinding_p.h" +#include "private/qdeclarativebinding_p.h" #include "qdeclarativecontext.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativeboundsignal_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativeboundsignal_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativedeclarativedata_p.h" -#include "qdeclarativestringconverters_p.h" -#include "qdeclarativelist_p.h" -#include "qdeclarativecompiler_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativestringconverters_p.h" +#include "private/qdeclarativelist_p.h" +#include "private/qdeclarativecompiler_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativeproperty_p.h b/src/declarative/qml/qdeclarativeproperty_p.h index 1bbee64..420a1ba 100644 --- a/src/declarative/qml/qdeclarativeproperty_p.h +++ b/src/declarative/qml/qdeclarativeproperty_p.h @@ -55,8 +55,8 @@ #include "qdeclarativeproperty.h" -#include "qdeclarativepropertycache_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativepropertycache_p.h" +#include "private/qdeclarativeguard_p.h" #include diff --git a/src/declarative/qml/qdeclarativepropertycache.cpp b/src/declarative/qml/qdeclarativepropertycache.cpp index fea59e5..111259d 100644 --- a/src/declarative/qml/qdeclarativepropertycache.cpp +++ b/src/declarative/qml/qdeclarativepropertycache.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativepropertycache_p.h" +#include "private/qdeclarativepropertycache_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativebinding_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativebinding_p.h" #include Q_DECLARE_METATYPE(QScriptValue); diff --git a/src/declarative/qml/qdeclarativepropertycache_p.h b/src/declarative/qml/qdeclarativepropertycache_p.h index bfbeff4..6b64a96 100644 --- a/src/declarative/qml/qdeclarativepropertycache_p.h +++ b/src/declarative/qml/qdeclarativepropertycache_p.h @@ -53,9 +53,9 @@ // We mean it. // -#include "qdeclarativerefcount_p.h" -#include "qdeclarativecleanup_p.h" -#include "qdeclarativenotifier_p.h" +#include "private/qdeclarativerefcount_p.h" +#include "private/qdeclarativecleanup_p.h" +#include "private/qdeclarativenotifier_p.h" #include diff --git a/src/declarative/qml/qdeclarativeproxymetaobject.cpp b/src/declarative/qml/qdeclarativeproxymetaobject.cpp index 1decea1..fe0dce7 100644 --- a/src/declarative/qml/qdeclarativeproxymetaobject.cpp +++ b/src/declarative/qml/qdeclarativeproxymetaobject.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeproxymetaobject_p.h" +#include "private/qdeclarativeproxymetaobject_p.h" #include diff --git a/src/declarative/qml/qdeclarativeproxymetaobject_p.h b/src/declarative/qml/qdeclarativeproxymetaobject_p.h index 3f103b6..3daa22f 100644 --- a/src/declarative/qml/qdeclarativeproxymetaobject_p.h +++ b/src/declarative/qml/qdeclarativeproxymetaobject_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qmetaobjectbuilder_p.h" +#include "private/qmetaobjectbuilder_p.h" #include "qdeclarative.h" #include diff --git a/src/declarative/qml/qdeclarativerefcount.cpp b/src/declarative/qml/qdeclarativerefcount.cpp index 943e35c..351fcf3 100644 --- a/src/declarative/qml/qdeclarativerefcount.cpp +++ b/src/declarative/qml/qdeclarativerefcount.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativerefcount_p.h" +#include "private/qdeclarativerefcount_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativerewrite.cpp b/src/declarative/qml/qdeclarativerewrite.cpp index afc6e63..3d40e77 100644 --- a/src/declarative/qml/qdeclarativerewrite.cpp +++ b/src/declarative/qml/qdeclarativerewrite.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativerewrite_p.h" +#include "private/qdeclarativerewrite_p.h" -#include "qdeclarativeglobal_p.h" +#include "private/qdeclarativeglobal_p.h" #include diff --git a/src/declarative/qml/qdeclarativescriptparser.cpp b/src/declarative/qml/qdeclarativescriptparser.cpp index 2b9cd4b..cba5bb9 100644 --- a/src/declarative/qml/qdeclarativescriptparser.cpp +++ b/src/declarative/qml/qdeclarativescriptparser.cpp @@ -39,16 +39,16 @@ ** ****************************************************************************/ -#include "qdeclarativescriptparser_p.h" +#include "private/qdeclarativescriptparser_p.h" -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include "parser/qdeclarativejsengine_p.h" #include "parser/qdeclarativejsparser_p.h" #include "parser/qdeclarativejslexer_p.h" #include "parser/qdeclarativejsnodepool_p.h" #include "parser/qdeclarativejsastvisitor_p.h" #include "parser/qdeclarativejsast_p.h" -#include "qdeclarativerewrite_p.h" +#include "private/qdeclarativerewrite_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativescriptparser_p.h b/src/declarative/qml/qdeclarativescriptparser_p.h index 68f1840..28e960b 100644 --- a/src/declarative/qml/qdeclarativescriptparser_p.h +++ b/src/declarative/qml/qdeclarativescriptparser_p.h @@ -53,7 +53,7 @@ // #include "qdeclarativeerror.h" -#include "qdeclarativeparser_p.h" +#include "private/qdeclarativeparser_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativesqldatabase.cpp b/src/declarative/qml/qdeclarativesqldatabase.cpp index 37d2d5b..45f277e 100644 --- a/src/declarative/qml/qdeclarativesqldatabase.cpp +++ b/src/declarative/qml/qdeclarativesqldatabase.cpp @@ -39,12 +39,12 @@ ** ****************************************************************************/ -#include "qdeclarativesqldatabase_p.h" +#include "private/qdeclarativesqldatabase_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativerefcount_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativerefcount_p.h" +#include "private/qdeclarativeengine_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativestringconverters.cpp b/src/declarative/qml/qdeclarativestringconverters.cpp index f3497e5..5c88b9a 100644 --- a/src/declarative/qml/qdeclarativestringconverters.cpp +++ b/src/declarative/qml/qdeclarativestringconverters.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativestringconverters_p.h" +#include "private/qdeclarativestringconverters_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativetypenamecache.cpp b/src/declarative/qml/qdeclarativetypenamecache.cpp index c4a8707..ea32aa0 100644 --- a/src/declarative/qml/qdeclarativetypenamecache.cpp +++ b/src/declarative/qml/qdeclarativetypenamecache.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativetypenamecache_p.h" +#include "private/qdeclarativetypenamecache_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativetypenamecache_p.h b/src/declarative/qml/qdeclarativetypenamecache_p.h index 3e24f5c..afd2fd9 100644 --- a/src/declarative/qml/qdeclarativetypenamecache_p.h +++ b/src/declarative/qml/qdeclarativetypenamecache_p.h @@ -53,8 +53,8 @@ // We mean it. // -#include "qdeclarativerefcount_p.h" -#include "qdeclarativecleanup_p.h" +#include "private/qdeclarativerefcount_p.h" +#include "private/qdeclarativecleanup_p.h" #include diff --git a/src/declarative/qml/qdeclarativetypenamescriptclass.cpp b/src/declarative/qml/qdeclarativetypenamescriptclass.cpp index 02370e0..9cac3e1 100644 --- a/src/declarative/qml/qdeclarativetypenamescriptclass.cpp +++ b/src/declarative/qml/qdeclarativetypenamescriptclass.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativetypenamescriptclass_p.h" +#include "private/qdeclarativetypenamescriptclass_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativetypenamecache_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativetypenamecache_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativetypenamescriptclass_p.h b/src/declarative/qml/qdeclarativetypenamescriptclass_p.h index b6e3f51..8e07f02 100644 --- a/src/declarative/qml/qdeclarativetypenamescriptclass_p.h +++ b/src/declarative/qml/qdeclarativetypenamescriptclass_p.h @@ -52,7 +52,7 @@ // // We mean it. // -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" #include diff --git a/src/declarative/qml/qdeclarativevaluetype.cpp b/src/declarative/qml/qdeclarativevaluetype.cpp index 00e6704..839e0dd 100644 --- a/src/declarative/qml/qdeclarativevaluetype.cpp +++ b/src/declarative/qml/qdeclarativevaluetype.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativevaluetype_p.h" +#include "private/qdeclarativevaluetype_p.h" -#include "qdeclarativemetatype_p.h" +#include "private/qdeclarativemetatype_p.h" #include diff --git a/src/declarative/qml/qdeclarativevaluetype_p.h b/src/declarative/qml/qdeclarativevaluetype_p.h index ad2f6c4..1fe8bd2 100644 --- a/src/declarative/qml/qdeclarativevaluetype_p.h +++ b/src/declarative/qml/qdeclarativevaluetype_p.h @@ -54,7 +54,7 @@ // #include "qdeclarativeproperty.h" -#include "qdeclarativeproperty_p.h" +#include "private/qdeclarativeproperty_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativevaluetypescriptclass.cpp b/src/declarative/qml/qdeclarativevaluetypescriptclass.cpp index a567c38..f7902b9 100644 --- a/src/declarative/qml/qdeclarativevaluetypescriptclass.cpp +++ b/src/declarative/qml/qdeclarativevaluetypescriptclass.cpp @@ -39,12 +39,12 @@ ** ****************************************************************************/ -#include "qdeclarativevaluetypescriptclass_p.h" +#include "private/qdeclarativevaluetypescriptclass_p.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativeproperty_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeguard_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativeproperty_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeguard_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativevme.cpp b/src/declarative/qml/qdeclarativevme.cpp index 2338bc3..5ba7f9b 100644 --- a/src/declarative/qml/qdeclarativevme.cpp +++ b/src/declarative/qml/qdeclarativevme.cpp @@ -39,26 +39,26 @@ ** ****************************************************************************/ -#include "qdeclarativevme_p.h" +#include "private/qdeclarativevme_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativeboundsignal_p.h" -#include "qdeclarativestringconverters_p.h" -#include "qmetaobjectbuilder_p.h" -#include "qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativeboundsignal_p.h" +#include "private/qdeclarativestringconverters_p.h" +#include "private/qmetaobjectbuilder_p.h" +#include "private/qdeclarativedeclarativedata_p.h" #include "qdeclarative.h" -#include "qdeclarativecustomparser_p.h" +#include "private/qdeclarativecustomparser_p.h" #include "qdeclarativeengine.h" #include "qdeclarativecontext.h" #include "qdeclarativecomponent.h" -#include "qdeclarativebinding_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativecomponent_p.h" -#include "qdeclarativevmemetaobject_p.h" -#include "qdeclarativebinding_p_p.h" -#include "qdeclarativecontext_p.h" -#include "qdeclarativecompiledbindings_p.h" -#include "qdeclarativeglobal_p.h" +#include "private/qdeclarativebinding_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativecomponent_p.h" +#include "private/qdeclarativevmemetaobject_p.h" +#include "private/qdeclarativebinding_p_p.h" +#include "private/qdeclarativecontext_p.h" +#include "private/qdeclarativecompiledbindings_p.h" +#include "private/qdeclarativeglobal_p.h" #include "qdeclarativescriptstring.h" #include diff --git a/src/declarative/qml/qdeclarativevme_p.h b/src/declarative/qml/qdeclarativevme_p.h index 1c6fd3c..87a1aab 100644 --- a/src/declarative/qml/qdeclarativevme_p.h +++ b/src/declarative/qml/qdeclarativevme_p.h @@ -54,7 +54,7 @@ // #include "qdeclarativeerror.h" -#include "qbitfield_p.h" +#include "private/qbitfield_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativevmemetaobject.cpp b/src/declarative/qml/qdeclarativevmemetaobject.cpp index 9a7bdd6..2644ecf 100644 --- a/src/declarative/qml/qdeclarativevmemetaobject.cpp +++ b/src/declarative/qml/qdeclarativevmemetaobject.cpp @@ -39,13 +39,13 @@ ** ****************************************************************************/ -#include "qdeclarativevmemetaobject_p.h" +#include "private/qdeclarativevmemetaobject_p.h" #include "qdeclarative.h" -#include "qdeclarativerefcount_p.h" +#include "private/qdeclarativerefcount_p.h" #include "qdeclarativeexpression.h" -#include "qdeclarativeexpression_p.h" -#include "qdeclarativecontext_p.h" +#include "private/qdeclarativeexpression_p.h" +#include "private/qdeclarativecontext_p.h" Q_DECLARE_METATYPE(QScriptValue); diff --git a/src/declarative/qml/qdeclarativevmemetaobject_p.h b/src/declarative/qml/qdeclarativevmemetaobject_p.h index 839f0cd..f13dd34 100644 --- a/src/declarative/qml/qdeclarativevmemetaobject_p.h +++ b/src/declarative/qml/qdeclarativevmemetaobject_p.h @@ -65,9 +65,9 @@ #include -#include "qdeclarativeguard_p.h" -#include "qdeclarativecompiler_p.h" -#include "qdeclarativecontext_p.h" +#include "private/qdeclarativeguard_p.h" +#include "private/qdeclarativecompiler_p.h" +#include "private/qdeclarativecontext_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/qml/qdeclarativewatcher.cpp b/src/declarative/qml/qdeclarativewatcher.cpp index 2f61195..9ea84b8 100644 --- a/src/declarative/qml/qdeclarativewatcher.cpp +++ b/src/declarative/qml/qdeclarativewatcher.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativewatcher_p.h" +#include "private/qdeclarativewatcher_p.h" #include "qdeclarativeexpression.h" #include "qdeclarativecontext.h" diff --git a/src/declarative/qml/qdeclarativeworkerscript.cpp b/src/declarative/qml/qdeclarativeworkerscript.cpp index 628681f..ddb0ece 100644 --- a/src/declarative/qml/qdeclarativeworkerscript.cpp +++ b/src/declarative/qml/qdeclarativeworkerscript.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativeworkerscript_p.h" -#include "qdeclarativelistmodel_p.h" -#include "qdeclarativelistmodelworkeragent_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeworkerscript_p.h" +#include "private/qdeclarativelistmodel_p.h" +#include "private/qdeclarativelistmodelworkeragent_p.h" +#include "private/qdeclarativeengine_p.h" #include #include diff --git a/src/declarative/qml/qdeclarativexmlhttprequest.cpp b/src/declarative/qml/qdeclarativexmlhttprequest.cpp index 58e67fa..be2a1a7 100644 --- a/src/declarative/qml/qdeclarativexmlhttprequest.cpp +++ b/src/declarative/qml/qdeclarativexmlhttprequest.cpp @@ -39,13 +39,13 @@ ** ****************************************************************************/ -#include "qdeclarativexmlhttprequest_p.h" +#include "private/qdeclarativexmlhttprequest_p.h" #include "qdeclarativeengine.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativerefcount_p.h" -#include "qdeclarativeengine_p.h" -#include "qdeclarativeexpression_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativerefcount_p.h" +#include "private/qdeclarativeengine_p.h" +#include "private/qdeclarativeexpression_p.h" #include #include diff --git a/src/declarative/qml/qmetaobjectbuilder.cpp b/src/declarative/qml/qmetaobjectbuilder.cpp index a518b03..6e4d7b8 100644 --- a/src/declarative/qml/qmetaobjectbuilder.cpp +++ b/src/declarative/qml/qmetaobjectbuilder.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qmetaobjectbuilder_p.h" +#include "private/qmetaobjectbuilder_p.h" #ifndef Q_OS_WIN #include diff --git a/src/declarative/qml/rewriter/rewriter.cpp b/src/declarative/qml/rewriter/rewriter.cpp index b458a8d..6d2ccfb 100644 --- a/src/declarative/qml/rewriter/rewriter.cpp +++ b/src/declarative/qml/rewriter/rewriter.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "rewriter_p.h" +#include "private/rewriter_p.h" #include diff --git a/src/declarative/qml/rewriter/rewriter_p.h b/src/declarative/qml/rewriter/rewriter_p.h index d1445d8..4799469 100644 --- a/src/declarative/qml/rewriter/rewriter_p.h +++ b/src/declarative/qml/rewriter/rewriter_p.h @@ -42,7 +42,7 @@ #ifndef REWRITER_H #define REWRITER_H -#include "textwriter_p.h" +#include "private/textwriter_p.h" #include diff --git a/src/declarative/qml/rewriter/textwriter.cpp b/src/declarative/qml/rewriter/textwriter.cpp index e63d24c..619f5c9 100644 --- a/src/declarative/qml/rewriter/textwriter.cpp +++ b/src/declarative/qml/rewriter/textwriter.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "textwriter_p.h" +#include "private/textwriter_p.h" QT_QML_BEGIN_NAMESPACE diff --git a/src/declarative/util/qdeclarativeanimation.cpp b/src/declarative/util/qdeclarativeanimation.cpp index ba1444e..33ddb46 100644 --- a/src/declarative/util/qdeclarativeanimation.cpp +++ b/src/declarative/util/qdeclarativeanimation.cpp @@ -39,11 +39,11 @@ ** ****************************************************************************/ -#include "qdeclarativeanimation_p.h" -#include "qdeclarativeanimation_p_p.h" +#include "private/qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p_p.h" -#include "qdeclarativebehavior_p.h" -#include "qdeclarativestateoperations_p.h" +#include "private/qdeclarativebehavior_p.h" +#include "private/qdeclarativestateoperations_p.h" #include #include diff --git a/src/declarative/util/qdeclarativeanimation_p.h b/src/declarative/util/qdeclarativeanimation_p.h index f620786..c839c2c 100644 --- a/src/declarative/util/qdeclarativeanimation_p.h +++ b/src/declarative/util/qdeclarativeanimation_p.h @@ -42,8 +42,8 @@ #ifndef QDECLARATIVEANIMATION_H #define QDECLARATIVEANIMATION_H -#include "qdeclarativetransition_p.h" -#include "qdeclarativestate_p.h" +#include "private/qdeclarativetransition_p.h" +#include "private/qdeclarativestate_p.h" #include #include diff --git a/src/declarative/util/qdeclarativeanimation_p_p.h b/src/declarative/util/qdeclarativeanimation_p_p.h index 8bcbeff..3b0f52e 100644 --- a/src/declarative/util/qdeclarativeanimation_p_p.h +++ b/src/declarative/util/qdeclarativeanimation_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p.h" -#include "qdeclarativenullablevalue_p_p.h" -#include "qdeclarativetimeline_p_p.h" +#include "private/qdeclarativenullablevalue_p_p.h" +#include "private/qdeclarativetimeline_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativebehavior.cpp b/src/declarative/util/qdeclarativebehavior.cpp index 87d6836..a6c578e 100644 --- a/src/declarative/util/qdeclarativebehavior.cpp +++ b/src/declarative/util/qdeclarativebehavior.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativebehavior_p.h" +#include "private/qdeclarativebehavior_p.h" -#include "qdeclarativeanimation_p.h" -#include "qdeclarativetransition_p.h" +#include "private/qdeclarativeanimation_p.h" +#include "private/qdeclarativetransition_p.h" #include #include diff --git a/src/declarative/util/qdeclarativebehavior_p.h b/src/declarative/util/qdeclarativebehavior_p.h index ff58210..e8a809f 100644 --- a/src/declarative/util/qdeclarativebehavior_p.h +++ b/src/declarative/util/qdeclarativebehavior_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEBEHAVIOR_H #define QDECLARATIVEBEHAVIOR_H -#include "qdeclarativestate_p.h" +#include "private/qdeclarativestate_p.h" #include #include diff --git a/src/declarative/util/qdeclarativebind.cpp b/src/declarative/util/qdeclarativebind.cpp index 26baa38..b7bd4e8 100644 --- a/src/declarative/util/qdeclarativebind.cpp +++ b/src/declarative/util/qdeclarativebind.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativebind_p.h" +#include "private/qdeclarativebind_p.h" -#include "qdeclarativenullablevalue_p_p.h" +#include "private/qdeclarativenullablevalue_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativeconnections.cpp b/src/declarative/util/qdeclarativeconnections.cpp index 0b9e3ab..596b306 100644 --- a/src/declarative/util/qdeclarativeconnections.cpp +++ b/src/declarative/util/qdeclarativeconnections.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativeconnections_p.h" +#include "private/qdeclarativeconnections_p.h" #include #include diff --git a/src/declarative/util/qdeclarativefontloader.cpp b/src/declarative/util/qdeclarativefontloader.cpp index bf98d40..b577b81 100644 --- a/src/declarative/util/qdeclarativefontloader.cpp +++ b/src/declarative/util/qdeclarativefontloader.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativefontloader_p.h" +#include "private/qdeclarativefontloader_p.h" #include #include diff --git a/src/declarative/util/qdeclarativelistaccessor.cpp b/src/declarative/util/qdeclarativelistaccessor.cpp index 4e56909..4ac587f 100644 --- a/src/declarative/util/qdeclarativelistaccessor.cpp +++ b/src/declarative/util/qdeclarativelistaccessor.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativelistaccessor_p.h" +#include "private/qdeclarativelistaccessor_p.h" #include diff --git a/src/declarative/util/qdeclarativelistmodel.cpp b/src/declarative/util/qdeclarativelistmodel.cpp index 13662c5..ec23bb2 100644 --- a/src/declarative/util/qdeclarativelistmodel.cpp +++ b/src/declarative/util/qdeclarativelistmodel.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativelistmodel_p_p.h" -#include "qdeclarativelistmodelworkeragent_p.h" -#include "qdeclarativeopenmetaobject_p.h" +#include "private/qdeclarativelistmodel_p_p.h" +#include "private/qdeclarativelistmodelworkeragent_p.h" +#include "private/qdeclarativeopenmetaobject_p.h" #include #include diff --git a/src/declarative/util/qdeclarativelistmodel_p_p.h b/src/declarative/util/qdeclarativelistmodel_p_p.h index 8041561..532eefa 100644 --- a/src/declarative/util/qdeclarativelistmodel_p_p.h +++ b/src/declarative/util/qdeclarativelistmodel_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativelistmodel_p.h" +#include "private/qdeclarativelistmodel_p.h" #include "qdeclarative.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativeengine_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/util/qdeclarativelistmodelworkeragent.cpp b/src/declarative/util/qdeclarativelistmodelworkeragent.cpp index 0e38632..3c0a086 100644 --- a/src/declarative/util/qdeclarativelistmodelworkeragent.cpp +++ b/src/declarative/util/qdeclarativelistmodelworkeragent.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativelistmodelworkeragent_p.h" -#include "qdeclarativelistmodel_p_p.h" -#include "qdeclarativedeclarativedata_p.h" -#include "qdeclarativeengine_p.h" +#include "private/qdeclarativelistmodelworkeragent_p.h" +#include "private/qdeclarativelistmodel_p_p.h" +#include "private/qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativeengine_p.h" #include "qdeclarativeinfo.h" #include diff --git a/src/declarative/util/qdeclarativeopenmetaobject.cpp b/src/declarative/util/qdeclarativeopenmetaobject.cpp index 6611885..3d95125 100644 --- a/src/declarative/util/qdeclarativeopenmetaobject.cpp +++ b/src/declarative/util/qdeclarativeopenmetaobject.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativeopenmetaobject_p.h" -#include "qdeclarativepropertycache_p.h" -#include "qdeclarativedeclarativedata_p.h" +#include "private/qdeclarativeopenmetaobject_p.h" +#include "private/qdeclarativepropertycache_p.h" +#include "private/qdeclarativedeclarativedata_p.h" #include #include diff --git a/src/declarative/util/qdeclarativepackage.cpp b/src/declarative/util/qdeclarativepackage.cpp index d144777..ac0788f 100644 --- a/src/declarative/util/qdeclarativepackage.cpp +++ b/src/declarative/util/qdeclarativepackage.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativepackage_p.h" +#include "private/qdeclarativepackage_p.h" #include #include diff --git a/src/declarative/util/qdeclarativepixmapcache.cpp b/src/declarative/util/qdeclarativepixmapcache.cpp index 1da929e..e57d3c2 100644 --- a/src/declarative/util/qdeclarativepixmapcache.cpp +++ b/src/declarative/util/qdeclarativepixmapcache.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativepixmapcache_p.h" +#include "private/qdeclarativepixmapcache_p.h" #include "qdeclarativenetworkaccessmanagerfactory.h" #include "qdeclarativeimageprovider.h" diff --git a/src/declarative/util/qdeclarativepropertychanges.cpp b/src/declarative/util/qdeclarativepropertychanges.cpp index 6ceec5d..e7228cc 100644 --- a/src/declarative/util/qdeclarativepropertychanges.cpp +++ b/src/declarative/util/qdeclarativepropertychanges.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativepropertychanges_p.h" +#include "private/qdeclarativepropertychanges_p.h" -#include "qdeclarativeopenmetaobject_p.h" +#include "private/qdeclarativeopenmetaobject_p.h" #include #include diff --git a/src/declarative/util/qdeclarativepropertychanges_p.h b/src/declarative/util/qdeclarativepropertychanges_p.h index 1274b0c..c2ed466 100644 --- a/src/declarative/util/qdeclarativepropertychanges_p.h +++ b/src/declarative/util/qdeclarativepropertychanges_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEPROPERTYCHANGES_H #define QDECLARATIVEPROPERTYCHANGES_H -#include "qdeclarativestateoperations_p.h" +#include "private/qdeclarativestateoperations_p.h" #include QT_BEGIN_HEADER diff --git a/src/declarative/util/qdeclarativepropertymap.cpp b/src/declarative/util/qdeclarativepropertymap.cpp index 0bdd91b..0e477b1 100644 --- a/src/declarative/util/qdeclarativepropertymap.cpp +++ b/src/declarative/util/qdeclarativepropertymap.cpp @@ -42,7 +42,7 @@ #include "qdeclarativepropertymap.h" #include -#include "qdeclarativeopenmetaobject_p.h" +#include "private/qdeclarativeopenmetaobject_p.h" #include diff --git a/src/declarative/util/qdeclarativesmoothedanimation.cpp b/src/declarative/util/qdeclarativesmoothedanimation.cpp index b3a9b9a..3411642 100644 --- a/src/declarative/util/qdeclarativesmoothedanimation.cpp +++ b/src/declarative/util/qdeclarativesmoothedanimation.cpp @@ -39,15 +39,15 @@ ** ****************************************************************************/ -#include "qdeclarativesmoothedanimation_p.h" -#include "qdeclarativesmoothedanimation_p_p.h" +#include "private/qdeclarativesmoothedanimation_p.h" +#include "private/qdeclarativesmoothedanimation_p_p.h" -#include "qdeclarativeanimation_p_p.h" +#include "private/qdeclarativeanimation_p_p.h" #include -#include "qdeclarativeproperty_p.h" +#include "private/qdeclarativeproperty_p.h" -#include "qdeclarativeglobal_p.h" +#include "private/qdeclarativeglobal_p.h" #include diff --git a/src/declarative/util/qdeclarativesmoothedanimation_p.h b/src/declarative/util/qdeclarativesmoothedanimation_p.h index 50ed00c..df53104 100644 --- a/src/declarative/util/qdeclarativesmoothedanimation_p.h +++ b/src/declarative/util/qdeclarativesmoothedanimation_p.h @@ -43,7 +43,7 @@ #define QDECLARATIVESMOOTHEDANIMATION_H #include -#include "qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p.h" #include diff --git a/src/declarative/util/qdeclarativesmoothedanimation_p_p.h b/src/declarative/util/qdeclarativesmoothedanimation_p_p.h index bdceeb3..81cd229 100644 --- a/src/declarative/util/qdeclarativesmoothedanimation_p_p.h +++ b/src/declarative/util/qdeclarativesmoothedanimation_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativesmoothedanimation_p.h" -#include "qdeclarativeanimation_p.h" +#include "private/qdeclarativesmoothedanimation_p.h" +#include "private/qdeclarativeanimation_p.h" -#include "qdeclarativeanimation_p_p.h" +#include "private/qdeclarativeanimation_p_p.h" #include diff --git a/src/declarative/util/qdeclarativespringfollow.cpp b/src/declarative/util/qdeclarativespringfollow.cpp index 063c3df..c42261d 100644 --- a/src/declarative/util/qdeclarativespringfollow.cpp +++ b/src/declarative/util/qdeclarativespringfollow.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativespringfollow_p.h" +#include "private/qdeclarativespringfollow_p.h" -#include "qdeclarativeanimation_p_p.h" +#include "private/qdeclarativeanimation_p_p.h" #include diff --git a/src/declarative/util/qdeclarativestate.cpp b/src/declarative/util/qdeclarativestate.cpp index 802ff1c..5e6c35e 100644 --- a/src/declarative/util/qdeclarativestate.cpp +++ b/src/declarative/util/qdeclarativestate.cpp @@ -39,14 +39,14 @@ ** ****************************************************************************/ -#include "qdeclarativestate_p_p.h" -#include "qdeclarativestate_p.h" - -#include "qdeclarativetransition_p.h" -#include "qdeclarativestategroup_p.h" -#include "qdeclarativestateoperations_p.h" -#include "qdeclarativeanimation_p.h" -#include "qdeclarativeanimation_p_p.h" +#include "private/qdeclarativestate_p_p.h" +#include "private/qdeclarativestate_p.h" + +#include "private/qdeclarativetransition_p.h" +#include "private/qdeclarativestategroup_p.h" +#include "private/qdeclarativestateoperations_p.h" +#include "private/qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativestate_p_p.h b/src/declarative/util/qdeclarativestate_p_p.h index 558c99b..4a2af0f 100644 --- a/src/declarative/util/qdeclarativestate_p_p.h +++ b/src/declarative/util/qdeclarativestate_p_p.h @@ -53,10 +53,10 @@ // We mean it. // -#include "qdeclarativestate_p.h" +#include "private/qdeclarativestate_p.h" -#include "qdeclarativeanimation_p_p.h" -#include "qdeclarativetransitionmanager_p_p.h" +#include "private/qdeclarativeanimation_p_p.h" +#include "private/qdeclarativetransitionmanager_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativestategroup.cpp b/src/declarative/util/qdeclarativestategroup.cpp index 258a9e9..ff78c60 100644 --- a/src/declarative/util/qdeclarativestategroup.cpp +++ b/src/declarative/util/qdeclarativestategroup.cpp @@ -39,10 +39,10 @@ ** ****************************************************************************/ -#include "qdeclarativestategroup_p.h" +#include "private/qdeclarativestategroup_p.h" -#include "qdeclarativetransition_p.h" -#include "qdeclarativestate_p_p.h" +#include "private/qdeclarativetransition_p.h" +#include "private/qdeclarativestate_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativestategroup_p.h b/src/declarative/util/qdeclarativestategroup_p.h index 9650199..cac23f4 100644 --- a/src/declarative/util/qdeclarativestategroup_p.h +++ b/src/declarative/util/qdeclarativestategroup_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVESTATEGROUP_H #define QDECLARATIVESTATEGROUP_H -#include "qdeclarativestate_p.h" +#include "private/qdeclarativestate_p.h" QT_BEGIN_HEADER diff --git a/src/declarative/util/qdeclarativestateoperations.cpp b/src/declarative/util/qdeclarativestateoperations.cpp index 3469136..2cc1fcc 100644 --- a/src/declarative/util/qdeclarativestateoperations.cpp +++ b/src/declarative/util/qdeclarativestateoperations.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativestateoperations_p.h" +#include "private/qdeclarativestateoperations_p.h" #include #include diff --git a/src/declarative/util/qdeclarativestateoperations_p.h b/src/declarative/util/qdeclarativestateoperations_p.h index c97b2da..d49ec63 100644 --- a/src/declarative/util/qdeclarativestateoperations_p.h +++ b/src/declarative/util/qdeclarativestateoperations_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVESTATEOPERATIONS_H #define QDECLARATIVESTATEOPERATIONS_H -#include "qdeclarativestate_p.h" +#include "private/qdeclarativestate_p.h" #include #include diff --git a/src/declarative/util/qdeclarativestyledtext.cpp b/src/declarative/util/qdeclarativestyledtext.cpp index 461de12..babd71b 100644 --- a/src/declarative/util/qdeclarativestyledtext.cpp +++ b/src/declarative/util/qdeclarativestyledtext.cpp @@ -45,7 +45,7 @@ #include #include #include -#include "qdeclarativestyledtext_p.h" +#include "private/qdeclarativestyledtext_p.h" /* QDeclarativeStyledText supports few tags: diff --git a/src/declarative/util/qdeclarativesystempalette.cpp b/src/declarative/util/qdeclarativesystempalette.cpp index d819c27..9bb3f69 100644 --- a/src/declarative/util/qdeclarativesystempalette.cpp +++ b/src/declarative/util/qdeclarativesystempalette.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativesystempalette_p.h" +#include "private/qdeclarativesystempalette_p.h" #include diff --git a/src/declarative/util/qdeclarativetimeline.cpp b/src/declarative/util/qdeclarativetimeline.cpp index 8e133b6..656c62b 100644 --- a/src/declarative/util/qdeclarativetimeline.cpp +++ b/src/declarative/util/qdeclarativetimeline.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativetimeline_p_p.h" +#include "private/qdeclarativetimeline_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativetimer.cpp b/src/declarative/util/qdeclarativetimer.cpp index 104e3ae..76e6d5e 100644 --- a/src/declarative/util/qdeclarativetimer.cpp +++ b/src/declarative/util/qdeclarativetimer.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativetimer_p.h" +#include "private/qdeclarativetimer_p.h" #include #include diff --git a/src/declarative/util/qdeclarativetransition.cpp b/src/declarative/util/qdeclarativetransition.cpp index 1e8be7f..815dc4c 100644 --- a/src/declarative/util/qdeclarativetransition.cpp +++ b/src/declarative/util/qdeclarativetransition.cpp @@ -39,13 +39,13 @@ ** ****************************************************************************/ -#include "qdeclarativestate_p.h" -#include "qdeclarativestategroup_p.h" -#include "qdeclarativestate_p_p.h" -#include "qdeclarativestateoperations_p.h" -#include "qdeclarativeanimation_p.h" -#include "qdeclarativeanimation_p_p.h" -#include "qdeclarativetransitionmanager_p_p.h" +#include "private/qdeclarativestate_p.h" +#include "private/qdeclarativestategroup_p.h" +#include "private/qdeclarativestate_p_p.h" +#include "private/qdeclarativestateoperations_p.h" +#include "private/qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p_p.h" +#include "private/qdeclarativetransitionmanager_p_p.h" #include diff --git a/src/declarative/util/qdeclarativetransition_p.h b/src/declarative/util/qdeclarativetransition_p.h index 2f9e7b5..fc7a577 100644 --- a/src/declarative/util/qdeclarativetransition_p.h +++ b/src/declarative/util/qdeclarativetransition_p.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVETRANSITION_H #define QDECLARATIVETRANSITION_H -#include "qdeclarativestate_p.h" +#include "private/qdeclarativestate_p.h" #include diff --git a/src/declarative/util/qdeclarativetransitionmanager.cpp b/src/declarative/util/qdeclarativetransitionmanager.cpp index f07fb23..bc40377 100644 --- a/src/declarative/util/qdeclarativetransitionmanager.cpp +++ b/src/declarative/util/qdeclarativetransitionmanager.cpp @@ -39,9 +39,9 @@ ** ****************************************************************************/ -#include "qdeclarativetransitionmanager_p_p.h" +#include "private/qdeclarativetransitionmanager_p_p.h" -#include "qdeclarativestate_p_p.h" +#include "private/qdeclarativestate_p_p.h" #include #include diff --git a/src/declarative/util/qdeclarativetransitionmanager_p_p.h b/src/declarative/util/qdeclarativetransitionmanager_p_p.h index d6e626e..4131391 100644 --- a/src/declarative/util/qdeclarativetransitionmanager_p_p.h +++ b/src/declarative/util/qdeclarativetransitionmanager_p_p.h @@ -53,7 +53,7 @@ // We mean it. // -#include "qdeclarativestateoperations_p.h" +#include "private/qdeclarativestateoperations_p.h" QT_BEGIN_NAMESPACE diff --git a/src/declarative/util/qdeclarativeutilmodule.cpp b/src/declarative/util/qdeclarativeutilmodule.cpp index 7f2f64a..d4c72bd 100644 --- a/src/declarative/util/qdeclarativeutilmodule.cpp +++ b/src/declarative/util/qdeclarativeutilmodule.cpp @@ -39,36 +39,36 @@ ** ****************************************************************************/ -#include "qdeclarativeutilmodule_p.h" -#include "qdeclarativeanimation_p.h" -#include "qdeclarativeanimation_p_p.h" -#include "qdeclarativebehavior_p.h" -#include "qdeclarativebind_p.h" -#include "qdeclarativeconnections_p.h" -#include "qdeclarativesmoothedanimation_p.h" -#include "qdeclarativefontloader_p.h" -#include "qdeclarativelistaccessor_p.h" -#include "qdeclarativelistmodel_p.h" -#include "qdeclarativenullablevalue_p_p.h" -#include "qdeclarativeopenmetaobject_p.h" -#include "qdeclarativepackage_p.h" -#include "qdeclarativepixmapcache_p.h" -#include "qdeclarativepropertychanges_p.h" +#include "private/qdeclarativeutilmodule_p.h" +#include "private/qdeclarativeanimation_p.h" +#include "private/qdeclarativeanimation_p_p.h" +#include "private/qdeclarativebehavior_p.h" +#include "private/qdeclarativebind_p.h" +#include "private/qdeclarativeconnections_p.h" +#include "private/qdeclarativesmoothedanimation_p.h" +#include "private/qdeclarativefontloader_p.h" +#include "private/qdeclarativelistaccessor_p.h" +#include "private/qdeclarativelistmodel_p.h" +#include "private/qdeclarativenullablevalue_p_p.h" +#include "private/qdeclarativeopenmetaobject_p.h" +#include "private/qdeclarativepackage_p.h" +#include "private/qdeclarativepixmapcache_p.h" +#include "private/qdeclarativepropertychanges_p.h" #include "qdeclarativepropertymap.h" -#include "qdeclarativespringfollow_p.h" -#include "qdeclarativestategroup_p.h" -#include "qdeclarativestateoperations_p.h" -#include "qdeclarativestate_p.h" -#include "qdeclarativestate_p_p.h" -#include "qdeclarativestyledtext_p.h" -#include "qdeclarativesystempalette_p.h" -#include "qdeclarativetimeline_p_p.h" -#include "qdeclarativetimer_p.h" -#include "qdeclarativetransitionmanager_p_p.h" -#include "qdeclarativetransition_p.h" +#include "private/qdeclarativespringfollow_p.h" +#include "private/qdeclarativestategroup_p.h" +#include "private/qdeclarativestateoperations_p.h" +#include "private/qdeclarativestate_p.h" +#include "private/qdeclarativestate_p_p.h" +#include "private/qdeclarativestyledtext_p.h" +#include "private/qdeclarativesystempalette_p.h" +#include "private/qdeclarativetimeline_p_p.h" +#include "private/qdeclarativetimer_p.h" +#include "private/qdeclarativetransitionmanager_p_p.h" +#include "private/qdeclarativetransition_p.h" #include "qdeclarativeview.h" #ifndef QT_NO_XMLPATTERNS -#include "qdeclarativexmllistmodel_p.h" +#include "private/qdeclarativexmllistmodel_p.h" #endif template diff --git a/src/declarative/util/qdeclarativexmllistmodel.cpp b/src/declarative/util/qdeclarativexmllistmodel.cpp index efc614b..b33af06 100644 --- a/src/declarative/util/qdeclarativexmllistmodel.cpp +++ b/src/declarative/util/qdeclarativexmllistmodel.cpp @@ -39,7 +39,7 @@ ** ****************************************************************************/ -#include "qdeclarativexmllistmodel_p.h" +#include "private/qdeclarativexmllistmodel_p.h" #include #include -- cgit v0.12 From 0be2652ce6e2703830551a9366c03b7062d5b506 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 2 Apr 2010 18:29:46 +0200 Subject: Fix QtDeclarative public headers to include All Qt public headers must do it like that. --- src/declarative/qml/qdeclarative.h | 10 +++++----- src/declarative/qml/qdeclarativecomponent.h | 4 ++-- src/declarative/qml/qdeclarativeexpression.h | 2 +- src/declarative/qml/qdeclarativeextensionplugin.h | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/declarative/qml/qdeclarative.h b/src/declarative/qml/qdeclarative.h index dfdef11..7c7f6e5 100644 --- a/src/declarative/qml/qdeclarative.h +++ b/src/declarative/qml/qdeclarative.h @@ -42,11 +42,11 @@ #ifndef QDECLARATIVE_H #define QDECLARATIVE_H -#include "qdeclarativeprivate.h" -#include "qdeclarativeparserstatus.h" -#include "qdeclarativepropertyvaluesource.h" -#include "qdeclarativepropertyvalueinterceptor.h" -#include "qdeclarativelist.h" +#include +#include +#include +#include +#include #include #include diff --git a/src/declarative/qml/qdeclarativecomponent.h b/src/declarative/qml/qdeclarativecomponent.h index 13a243e..6ee5070 100644 --- a/src/declarative/qml/qdeclarativecomponent.h +++ b/src/declarative/qml/qdeclarativecomponent.h @@ -42,8 +42,8 @@ #ifndef QDECLARATIVECOMPONENT_H #define QDECLARATIVECOMPONENT_H -#include "qdeclarative.h" -#include "qdeclarativeerror.h" +#include +#include #include #include diff --git a/src/declarative/qml/qdeclarativeexpression.h b/src/declarative/qml/qdeclarativeexpression.h index 73a5793..35d6949 100644 --- a/src/declarative/qml/qdeclarativeexpression.h +++ b/src/declarative/qml/qdeclarativeexpression.h @@ -42,7 +42,7 @@ #ifndef QDECLARATIVEEXPRESSION_H #define QDECLARATIVEEXPRESSION_H -#include "qdeclarativeerror.h" +#include #include #include diff --git a/src/declarative/qml/qdeclarativeextensionplugin.h b/src/declarative/qml/qdeclarativeextensionplugin.h index c2ad798..8095ec7 100644 --- a/src/declarative/qml/qdeclarativeextensionplugin.h +++ b/src/declarative/qml/qdeclarativeextensionplugin.h @@ -44,7 +44,7 @@ #include -#include "qdeclarativeextensioninterface.h" +#include QT_BEGIN_HEADER -- cgit v0.12 From 5b1fb5c69b72e954e32ae52d844996840bf43708 Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 2 Apr 2010 18:15:04 +0200 Subject: Autotest: Add QtDeclarative to the "headersclean" test All public includes are clean for QtDeclarative now. --- tests/auto/headers/headersclean.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/auto/headers/headersclean.cpp b/tests/auto/headers/headersclean.cpp index b932b9f..c98cd10 100644 --- a/tests/auto/headers/headersclean.cpp +++ b/tests/auto/headers/headersclean.cpp @@ -55,7 +55,7 @@ // extra #include -//#include +#include #include #include #include -- cgit v0.12 From ab7a2ade3393559ce7d7c1bed4ab7eedb7939c6b Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 2 Apr 2010 18:07:48 +0200 Subject: Fix some more Q_PROPERTY followed by semi-colon --- examples/declarative/extending/binding/person.h | 2 +- examples/declarative/extending/extended/lineedit.h | 8 ++++---- examples/declarative/extending/grouped/person.h | 2 +- examples/declarative/extending/signal/person.h | 2 +- examples/declarative/extending/valuesource/person.h | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/declarative/extending/binding/person.h b/examples/declarative/extending/binding/person.h index 1bec71c..2d4ec12 100644 --- a/examples/declarative/extending/binding/person.h +++ b/examples/declarative/extending/binding/person.h @@ -80,7 +80,7 @@ class Person : public QObject { Q_OBJECT Q_PROPERTY(QString name READ name WRITE setName NOTIFY nameChanged) // ![0] -Q_PROPERTY(ShoeDescription *shoe READ shoe CONSTANT); +Q_PROPERTY(ShoeDescription *shoe READ shoe CONSTANT) // ![0] public: Person(QObject *parent = 0); diff --git a/examples/declarative/extending/extended/lineedit.h b/examples/declarative/extending/extended/lineedit.h index ca96d05..9730b91 100644 --- a/examples/declarative/extending/extended/lineedit.h +++ b/examples/declarative/extending/extended/lineedit.h @@ -46,10 +46,10 @@ class LineEditExtension : public QObject { Q_OBJECT -Q_PROPERTY(int leftMargin READ leftMargin WRITE setLeftMargin NOTIFY marginsChanged); -Q_PROPERTY(int rightMargin READ rightMargin WRITE setRightMargin NOTIFY marginsChanged); -Q_PROPERTY(int topMargin READ topMargin WRITE setTopMargin NOTIFY marginsChanged); -Q_PROPERTY(int bottomMargin READ bottomMargin WRITE setBottomMargin NOTIFY marginsChanged); +Q_PROPERTY(int leftMargin READ leftMargin WRITE setLeftMargin NOTIFY marginsChanged) +Q_PROPERTY(int rightMargin READ rightMargin WRITE setRightMargin NOTIFY marginsChanged) +Q_PROPERTY(int topMargin READ topMargin WRITE setTopMargin NOTIFY marginsChanged) +Q_PROPERTY(int bottomMargin READ bottomMargin WRITE setBottomMargin NOTIFY marginsChanged) public: LineEditExtension(QObject *); diff --git a/examples/declarative/extending/grouped/person.h b/examples/declarative/extending/grouped/person.h index 5dab378..5ea2348 100644 --- a/examples/declarative/extending/grouped/person.h +++ b/examples/declarative/extending/grouped/person.h @@ -77,7 +77,7 @@ class Person : public QObject { Q_OBJECT Q_PROPERTY(QString name READ name WRITE setName) // ![1] -Q_PROPERTY(ShoeDescription *shoe READ shoe); +Q_PROPERTY(ShoeDescription *shoe READ shoe) // ![1] public: Person(QObject *parent = 0); diff --git a/examples/declarative/extending/signal/person.h b/examples/declarative/extending/signal/person.h index 0f86d8b..08caebf 100644 --- a/examples/declarative/extending/signal/person.h +++ b/examples/declarative/extending/signal/person.h @@ -76,7 +76,7 @@ QML_DECLARE_TYPE(ShoeDescription); class Person : public QObject { Q_OBJECT Q_PROPERTY(QString name READ name WRITE setName) -Q_PROPERTY(ShoeDescription *shoe READ shoe); +Q_PROPERTY(ShoeDescription *shoe READ shoe) public: Person(QObject *parent = 0); diff --git a/examples/declarative/extending/valuesource/person.h b/examples/declarative/extending/valuesource/person.h index 0f86d8b..08caebf 100644 --- a/examples/declarative/extending/valuesource/person.h +++ b/examples/declarative/extending/valuesource/person.h @@ -76,7 +76,7 @@ QML_DECLARE_TYPE(ShoeDescription); class Person : public QObject { Q_OBJECT Q_PROPERTY(QString name READ name WRITE setName) -Q_PROPERTY(ShoeDescription *shoe READ shoe); +Q_PROPERTY(ShoeDescription *shoe READ shoe) public: Person(QObject *parent = 0); -- cgit v0.12 From 779a0d678fd63d0d26c0c6bda7c3e82dcb8eb75d Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 2 Apr 2010 21:05:47 +0200 Subject: Fix compilation: Linguist uses QtDeclarative .cpp and private headers --- tools/linguist/lupdate/lupdate.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/linguist/lupdate/lupdate.pro b/tools/linguist/lupdate/lupdate.pro index 1f88931..3ed909a 100644 --- a/tools/linguist/lupdate/lupdate.pro +++ b/tools/linguist/lupdate/lupdate.pro @@ -16,7 +16,7 @@ include(../shared/formats.pri) include(../shared/proparser.pri) include($$QT_SOURCE_TREE/src/declarative/qml/parser/parser.pri) -INCLUDEPATH += $$QT_SOURCE_TREE/src/declarative/qml +INCLUDEPATH += $$QT_SOURCE_TREE/src/declarative/qml $$QT_BUILD_TREE/include/QtDeclarative SOURCES += \ main.cpp \ -- cgit v0.12 ----------------------------------- 4 files changed, 7 insertions(+), 79 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 66b9238..b5c4a7a 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -805,24 +805,6 @@ Classes and functions classes using multiple inheritance and their descendants will appear multiple times. - -.. function:: getargspec(func) - - Get the names and default values of a Python function's arguments. A - :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is - returned. *args* is a list of the argument names. *varargs* and *keywords* - are the names of the ``*`` and ``**`` arguments or ``None``. *defaults* is a - tuple of default argument values or ``None`` if there are no default - arguments; if this tuple has *n* elements, they correspond to the last - *n* elements listed in *args*. - - .. deprecated:: 3.0 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. This function will be removed - in Python 3.6. - - .. function:: getfullargspec(func) Get the names and default values of a Python function's arguments. A diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 375d94e..b6388d5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -145,7 +145,8 @@ Removed API and Feature Removals ------------------------ -* None yet. +* ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). + :func:`inspect.getfullargspec` is an almost drop in replacement. Porting to Python 3.6 diff --git a/Lib/inspect.py b/Lib/inspect.py index 24c8df7..4e78d80 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1002,31 +1002,6 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw - -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's arguments. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** arguments or None. - 'defaults' is an n-tuple of the default values of the last n arguments. - - Use the getfullargspec() API for Python 3 code, as annotations - and keyword arguments are supported. getargspec() will raise ValueError - if the func has either annotations or keyword arguments. - """ - warnings.warn("inspect.getargspec() is deprecated, " - "use inspect.signature() instead", DeprecationWarning, - stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only arguments or annotations" - ", use getfullargspec() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) - FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 042617b..bd0605e 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -629,18 +629,6 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) - def assertArgSpecEquals(self, routine, args_e, varargs_e=None, - varkw_e=None, defaults_e=None, formatted=None): - with self.assertWarns(DeprecationWarning): - args, varargs, varkw, defaults = inspect.getargspec(routine) - self.assertEqual(args, args_e) - self.assertEqual(varargs, varargs_e) - self.assertEqual(varkw, varkw_e) - self.assertEqual(defaults, defaults_e) - if formatted is not None: - self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), - formatted) - def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -659,23 +647,6 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) - def test_getargspec(self): - self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') - - self.assertArgSpecEquals(mod.spam, - ['a', 'b', 'c', 'd', 'e', 'f'], - 'g', 'h', (3, 4, 5), - '(a, b, c, d=3, e=4, f=5, *g, **h)') - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyworded, []) - - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.annotated, []) - self.assertRaises(ValueError, self.assertArgSpecEquals, - mod2.keyword_only_arg, []) - - def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -689,20 +660,19 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_argspec_api_ignores_wrapped(self): + def test_fullargspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check - self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -780,11 +750,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getargspec_method(self): + def test_getfullargspec_method(self): class A(object): def m(self): pass - self.assertArgSpecEquals(A.m, ['self']) + self.assertFullArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): -- cgit v0.12 From 6dfbc5d98eee400fa7b775b53c11a4e37fca3547 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:49:00 +0300 Subject: Issue #13248: Remove inspect.getmoduleinfo() from 3.6 (deprecated in 3.3) --- Doc/library/inspect.rst | 21 +-------------------- Doc/whatsnew/3.6.rst | 4 ++++ Lib/inspect.py | 17 ----------------- 3 files changed, 5 insertions(+), 37 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index b5c4a7a..1d0edea 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -227,24 +227,6 @@ attributes: listed in the metaclass' custom :meth:`__dir__`. -.. function:: getmoduleinfo(path) - - Returns a :term:`named tuple` ``ModuleInfo(name, suffix, mode, module_type)`` - of values that describe how Python will interpret the file identified by - *path* if it is a module, or ``None`` if it would not be identified as a - module. In that tuple, *name* is the name of the module without the name of - any enclosing package, *suffix* is the trailing part of the file name (which - may not be a dot-delimited extension), *mode* is the :func:`open` mode that - would be used (``'r'`` or ``'rb'``), and *module_type* is an integer giving - the type of the module. *module_type* will have a value which can be - compared to the constants defined in the :mod:`imp` module; see the - documentation for that module for more information on module types. - - .. deprecated:: 3.3 - You may check the file path's suffix against the supported suffixes - listed in :mod:`importlib.machinery` to infer the same information. - - .. function:: getmodulename(path) Return the name of the module named by the file *path*, without including the @@ -258,8 +240,7 @@ attributes: still return ``None``. .. versionchanged:: 3.3 - This function is now based directly on :mod:`importlib` rather than the - deprecated :func:`getmoduleinfo`. + The function is based directly on :mod:`importlib`. .. function:: ismodule(object) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b6388d5..c881d30 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -148,6 +148,10 @@ API and Feature Removals * ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). :func:`inspect.getfullargspec` is an almost drop in replacement. +* ``inspect.getmoduleinfo`` was removed (was deprecated since CPython 3.3). + :func:`inspect.getmodulename` should be used for obtaining the module + name for a given path. + Porting to Python 3.6 ===================== diff --git a/Lib/inspect.py b/Lib/inspect.py index 4e78d80..305aafe 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -623,23 +623,6 @@ def getfile(object): raise TypeError('{!r} is not a module, class, method, ' 'function, traceback, frame, or code object'.format(object)) -ModuleInfo = namedtuple('ModuleInfo', 'name suffix mode module_type') - -def getmoduleinfo(path): - """Get the module name, suffix, mode, and module type for a given file.""" - warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning, - 2) - with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) - import imp - filename = os.path.basename(path) - suffixes = [(-len(suffix), suffix, mode, mtype) - for suffix, mode, mtype in imp.get_suffixes()] - suffixes.sort() # try longest suffixes first, in case they overlap - for neglen, suffix, mode, mtype in suffixes: - if filename[neglen:] == suffix: - return ModuleInfo(filename[:neglen], suffix, mode, mtype) - def getmodulename(path): """Return the module name for a given file, or None.""" fname = os.path.basename(path) -- cgit v0.12 From 5661316e4d50a27666f206ef6483118f801a854e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 23 Jul 2015 17:51:34 +0300 Subject: Sync NEWS (re removed inspect.getargspec and inspect.getmoduleinfo) --- Doc/whatsnew/3.6.rst | 2 +- Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c881d30..3f83f19 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -148,7 +148,7 @@ API and Feature Removals * ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). :func:`inspect.getfullargspec` is an almost drop in replacement. -* ``inspect.getmoduleinfo`` was removed (was deprecated since CPython 3.3). +* ``inspect.getmoduleinfo()`` was removed (was deprecated since CPython 3.3). :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. diff --git a/Misc/NEWS b/Misc/NEWS index b68f8a8..d433b95 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Library - Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. + What's New in Python 3.5.0 beta 4? ================================== -- cgit v0.12 From 9f4b8605e624c249ad3d0f31d03f42c38d7ccdb2 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 24 Jul 2015 00:43:44 -0400 Subject: Issue #24300: Minor refactoring. --- Modules/nismodule.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/nismodule.c b/Modules/nismodule.c index 64eb5db..b6a855c 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -76,11 +76,7 @@ nis_mapname (char *map, int *pfix) *pfix = 0; for (i=0; aliases[i].alias != 0L; i++) { - if (!strcmp (aliases[i].alias, map)) { - *pfix = aliases[i].fix; - return aliases[i].map; - } - if (!strcmp (aliases[i].map, map)) { + if (!strcmp (aliases[i].alias, map) || !strcmp (aliases[i].map, map)) { *pfix = aliases[i].fix; return aliases[i].map; } -- cgit v0.12 From c94a1dc4c9a5d23fdb21f6928dd737a59ef3dfb2 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:43:13 +1200 Subject: - Issue #2091: error correctly on open() with mode 'U' and '+' open() accepted a 'U' mode string containing '+', but 'U' can only be used with 'r'. Patch from Jeff Balogh and John O'Connor. --- Lib/_pyio.py | 4 ++-- Lib/test/test_file.py | 2 +- Misc/NEWS | 3 +++ Modules/_io/_iomodule.c | 16 ++++++++-------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 50ad9ff..33d8a3f 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -181,8 +181,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, text = "t" in modes binary = "b" in modes if "U" in modes: - if creating or writing or appending: - raise ValueError("can't use U and writing mode at once") + if creating or writing or appending or updating: + raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") import warnings warnings.warn("'U' mode is deprecated", DeprecationWarning, 2) diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index d54e976..94f189a 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -139,7 +139,7 @@ class OtherFileTests: def testModeStrings(self): # check invalid mode strings - for mode in ("", "aU", "wU+"): + for mode in ("", "aU", "wU+", "U+", "+U", "rU+"): try: f = self.open(TESTFN, mode) except ValueError: diff --git a/Misc/NEWS b/Misc/NEWS index 81d5f30..f85eb16 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,9 @@ Library - Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + - Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 528bcd4..1c2d3a0 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -248,8 +248,8 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, _Py_IDENTIFIER(close); if (!PyUnicode_Check(file) && - !PyBytes_Check(file) && - !PyNumber_Check(file)) { + !PyBytes_Check(file) && + !PyNumber_Check(file)) { PyErr_Format(PyExc_TypeError, "invalid file: %R", file); return NULL; } @@ -307,9 +307,9 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, /* Parameters validation */ if (universal) { - if (writing || appending) { + if (creating || writing || appending || updating) { PyErr_SetString(PyExc_ValueError, - "can't use U and writing mode at once"); + "mode U cannot be combined with x', 'w', 'a', or '+'"); return NULL; } if (PyErr_WarnEx(PyExc_DeprecationWarning, @@ -437,10 +437,10 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, /* wraps into a TextIOWrapper */ wrapper = PyObject_CallFunction((PyObject *)&PyTextIOWrapper_Type, - "Osssi", - buffer, - encoding, errors, newline, - line_buffering); + "Osssi", + buffer, + encoding, errors, newline, + line_buffering); if (wrapper == NULL) goto error; result = wrapper; -- cgit v0.12 From aece8248b6257cd8c0840963a794537b9e1d9d14 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 26 Jul 2015 06:50:51 +1200 Subject: Issue #24710: Use cls in TracebackException.from_exception. Minor cleanup patch from Berker Peksag. --- Lib/traceback.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 02edeb6..3d2e5e0 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -477,10 +477,9 @@ class TracebackException: self._load_lines() @classmethod - def from_exception(self, exc, *args, **kwargs): + def from_exception(cls, exc, *args, **kwargs): """Create a TracebackException from an exception.""" - return TracebackException( - type(exc), exc, exc.__traceback__, *args, **kwargs) + return cls(type(exc), exc, exc.__traceback__, *args, **kwargs) def _load_lines(self): """Private API. force all lines in the stack to be loaded.""" -- cgit v0.12 From 870db9025f79ea47b1f93260b8714bc25fa7cc47 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 26 Jul 2015 13:11:49 +0200 Subject: Closes #20544: use specific asserts in operator tests. Patch by Serhiy. --- Lib/test/test_operator.py | 50 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index da9c8ef..54fd1f4 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -120,63 +120,63 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.add) self.assertRaises(TypeError, operator.add, None, None) - self.assertTrue(operator.add(3, 4) == 7) + self.assertEqual(operator.add(3, 4), 7) def test_bitwise_and(self): operator = self.module self.assertRaises(TypeError, operator.and_) self.assertRaises(TypeError, operator.and_, None, None) - self.assertTrue(operator.and_(0xf, 0xa) == 0xa) + self.assertEqual(operator.and_(0xf, 0xa), 0xa) def test_concat(self): operator = self.module self.assertRaises(TypeError, operator.concat) self.assertRaises(TypeError, operator.concat, None, None) - self.assertTrue(operator.concat('py', 'thon') == 'python') - self.assertTrue(operator.concat([1, 2], [3, 4]) == [1, 2, 3, 4]) - self.assertTrue(operator.concat(Seq1([5, 6]), Seq1([7])) == [5, 6, 7]) - self.assertTrue(operator.concat(Seq2([5, 6]), Seq2([7])) == [5, 6, 7]) + self.assertEqual(operator.concat('py', 'thon'), 'python') + self.assertEqual(operator.concat([1, 2], [3, 4]), [1, 2, 3, 4]) + self.assertEqual(operator.concat(Seq1([5, 6]), Seq1([7])), [5, 6, 7]) + self.assertEqual(operator.concat(Seq2([5, 6]), Seq2([7])), [5, 6, 7]) self.assertRaises(TypeError, operator.concat, 13, 29) def test_countOf(self): operator = self.module self.assertRaises(TypeError, operator.countOf) self.assertRaises(TypeError, operator.countOf, None, None) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 3) == 1) - self.assertTrue(operator.countOf([1, 2, 1, 3, 1, 4], 5) == 0) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 3), 1) + self.assertEqual(operator.countOf([1, 2, 1, 3, 1, 4], 5), 0) def test_delitem(self): operator = self.module a = [4, 3, 2, 1] self.assertRaises(TypeError, operator.delitem, a) self.assertRaises(TypeError, operator.delitem, a, None) - self.assertTrue(operator.delitem(a, 1) is None) - self.assertTrue(a == [4, 2, 1]) + self.assertIsNone(operator.delitem(a, 1)) + self.assertEqual(a, [4, 2, 1]) def test_floordiv(self): operator = self.module self.assertRaises(TypeError, operator.floordiv, 5) self.assertRaises(TypeError, operator.floordiv, None, None) - self.assertTrue(operator.floordiv(5, 2) == 2) + self.assertEqual(operator.floordiv(5, 2), 2) def test_truediv(self): operator = self.module self.assertRaises(TypeError, operator.truediv, 5) self.assertRaises(TypeError, operator.truediv, None, None) - self.assertTrue(operator.truediv(5, 2) == 2.5) + self.assertEqual(operator.truediv(5, 2), 2.5) def test_getitem(self): operator = self.module a = range(10) self.assertRaises(TypeError, operator.getitem) self.assertRaises(TypeError, operator.getitem, a, None) - self.assertTrue(operator.getitem(a, 2) == 2) + self.assertEqual(operator.getitem(a, 2), 2) def test_indexOf(self): operator = self.module self.assertRaises(TypeError, operator.indexOf) self.assertRaises(TypeError, operator.indexOf, None, None) - self.assertTrue(operator.indexOf([4, 3, 2, 1], 3) == 1) + self.assertEqual(operator.indexOf([4, 3, 2, 1], 3), 1) self.assertRaises(ValueError, operator.indexOf, [4, 3, 2, 1], 0) def test_invert(self): @@ -189,21 +189,21 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.lshift) self.assertRaises(TypeError, operator.lshift, None, 42) - self.assertTrue(operator.lshift(5, 1) == 10) - self.assertTrue(operator.lshift(5, 0) == 5) + self.assertEqual(operator.lshift(5, 1), 10) + self.assertEqual(operator.lshift(5, 0), 5) self.assertRaises(ValueError, operator.lshift, 2, -1) def test_mod(self): operator = self.module self.assertRaises(TypeError, operator.mod) self.assertRaises(TypeError, operator.mod, None, 42) - self.assertTrue(operator.mod(5, 2) == 1) + self.assertEqual(operator.mod(5, 2), 1) def test_mul(self): operator = self.module self.assertRaises(TypeError, operator.mul) self.assertRaises(TypeError, operator.mul, None, None) - self.assertTrue(operator.mul(5, 2) == 10) + self.assertEqual(operator.mul(5, 2), 10) def test_matmul(self): operator = self.module @@ -227,7 +227,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.or_) self.assertRaises(TypeError, operator.or_, None, None) - self.assertTrue(operator.or_(0xa, 0x5) == 0xf) + self.assertEqual(operator.or_(0xa, 0x5), 0xf) def test_pos(self): operator = self.module @@ -250,8 +250,8 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.rshift) self.assertRaises(TypeError, operator.rshift, None, 42) - self.assertTrue(operator.rshift(5, 1) == 2) - self.assertTrue(operator.rshift(5, 0) == 5) + self.assertEqual(operator.rshift(5, 1), 2) + self.assertEqual(operator.rshift(5, 0), 5) self.assertRaises(ValueError, operator.rshift, 2, -1) def test_contains(self): @@ -266,15 +266,15 @@ class OperatorTestCase: a = list(range(3)) self.assertRaises(TypeError, operator.setitem, a) self.assertRaises(TypeError, operator.setitem, a, None, None) - self.assertTrue(operator.setitem(a, 0, 2) is None) - self.assertTrue(a == [2, 1, 2]) + self.assertIsNone(operator.setitem(a, 0, 2)) + self.assertEqual(a, [2, 1, 2]) self.assertRaises(IndexError, operator.setitem, a, 4, 2) def test_sub(self): operator = self.module self.assertRaises(TypeError, operator.sub) self.assertRaises(TypeError, operator.sub, None, None) - self.assertTrue(operator.sub(5, 2) == 3) + self.assertEqual(operator.sub(5, 2), 3) def test_truth(self): operator = self.module @@ -292,7 +292,7 @@ class OperatorTestCase: operator = self.module self.assertRaises(TypeError, operator.xor) self.assertRaises(TypeError, operator.xor, None, None) - self.assertTrue(operator.xor(0xb, 0xc) == 0x7) + self.assertEqual(operator.xor(0xb, 0xc), 0x7) def test_is(self): operator = self.module -- cgit v0.12 From c6d9228290c1635588d8edddc04f2b2278705867 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 28 Jul 2015 15:55:07 +1200 Subject: Issue #23426: run_setup was broken in distutils. Patch from Alexander Belopolsky. --- Lib/distutils/core.py | 5 ++--- Lib/distutils/tests/test_core.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/Lib/distutils/core.py b/Lib/distutils/core.py index f05b34b..d603d4a 100644 --- a/Lib/distutils/core.py +++ b/Lib/distutils/core.py @@ -204,16 +204,15 @@ def run_setup (script_name, script_args=None, stop_after="run"): global _setup_stop_after, _setup_distribution _setup_stop_after = stop_after - save_argv = sys.argv + save_argv = sys.argv.copy() g = {'__file__': script_name} - l = {} try: try: sys.argv[0] = script_name if script_args is not None: sys.argv[1:] = script_args with open(script_name, 'rb') as f: - exec(f.read(), g, l) + exec(f.read(), g) finally: sys.argv = save_argv _setup_stop_after = None diff --git a/Lib/distutils/tests/test_core.py b/Lib/distutils/tests/test_core.py index 41321f7..57856f1 100644 --- a/Lib/distutils/tests/test_core.py +++ b/Lib/distutils/tests/test_core.py @@ -28,6 +28,21 @@ from distutils.core import setup setup() """ +setup_does_nothing = """\ +from distutils.core import setup +setup() +""" + + +setup_defines_subclass = """\ +from distutils.core import setup +from distutils.command.install import install as _install + +class install(_install): + sub_commands = _install.sub_commands + ['cmd'] + +setup(cmdclass={'install': install}) +""" class CoreTestCase(support.EnvironGuard, unittest.TestCase): @@ -65,6 +80,21 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): distutils.core.run_setup( self.write_setup(setup_using___file__)) + def test_run_setup_preserves_sys_argv(self): + # Make sure run_setup does not clobber sys.argv + argv_copy = sys.argv.copy() + distutils.core.run_setup( + self.write_setup(setup_does_nothing)) + self.assertEqual(sys.argv, argv_copy) + + def test_run_setup_defines_subclass(self): + # Make sure the script can use __file__; if that's missing, the test + # setup.py script will raise NameError. + dist = distutils.core.run_setup( + self.write_setup(setup_defines_subclass)) + install = dist.get_command_obj('install') + self.assertIn('cmd', install.sub_commands) + def test_run_setup_uses_current_dir(self): # This tests that the setup script is run with the current directory # as its own current directory; this was temporarily broken by a diff --git a/Misc/NEWS b/Misc/NEWS index 2232357..3004539 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. + - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. - Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. -- cgit v0.12 From 76b1714be84ab6e73b9b0716cbb233f044318516 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 29 Jul 2015 23:51:47 +0300 Subject: Issue #24360: Improve __repr__ of argparse.Namespace() for invalid identifiers. Patch by Matthias Bussonnier. --- Lib/argparse.py | 8 +++++++- Lib/test/test_argparse.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 9a06719..cc53841 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -118,10 +118,16 @@ class _AttributeHolder(object): def __repr__(self): type_name = type(self).__name__ arg_strings = [] + star_args = {} for arg in self._get_args(): arg_strings.append(repr(arg)) for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) + if name.isidentifier(): + arg_strings.append('%s=%r' % (name, value)) + else: + star_args[name] = value + if star_args: + arg_strings.append('**%s' % repr(star_args)) return '%s(%s)' % (type_name, ', '.join(arg_strings)) def _get_kwargs(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 27bfad5..893ec39 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4512,6 +4512,21 @@ class TestStrings(TestCase): string = "Namespace(bar='spam', foo=42)" self.assertStringEqual(ns, string) + def test_namespace_starkwargs_notidentifier(self): + ns = argparse.Namespace(**{'"': 'quote'}) + string = """Namespace(**{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_kwargs_and_starkwargs_notidentifier(self): + ns = argparse.Namespace(a=1, **{'"': 'quote'}) + string = """Namespace(a=1, **{'"': 'quote'})""" + self.assertStringEqual(ns, string) + + def test_namespace_starkwargs_identifier(self): + ns = argparse.Namespace(**{'valid': True}) + string = "Namespace(valid=True)" + self.assertStringEqual(ns, string) + def test_parser(self): parser = argparse.ArgumentParser(prog='PROG') string = ( diff --git a/Misc/NEWS b/Misc/NEWS index 2c5d17a..a3ada97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,6 +13,9 @@ Core and Builtins Library ------- +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. -- cgit v0.12 From fa3922cfd0697bc3286c301c5d9458160c80cc89 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Fri, 31 Jul 2015 04:11:29 +0300 Subject: Issue #13248: Delete remaining references of inspect.getargspec(). Noticed by Yaroslav Halchenko. --- Doc/library/inspect.rst | 4 +--- Lib/inspect.py | 7 ++----- Lib/test/test_inspect.py | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 1e0996c..4357090 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -805,8 +805,6 @@ Classes and functions from kwonlyargs to defaults. *annotations* is a dictionary mapping argument names to annotations. - The first four items in the tuple correspond to :func:`getargspec`. - .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -835,7 +833,7 @@ Classes and functions .. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) Format a pretty argument spec from the values returned by - :func:`getargspec` or :func:`getfullargspec`. + :func:`getfullargspec`. The first seven arguments are (``args``, ``varargs``, ``varkw``, ``defaults``, ``kwonlyargs``, ``kwonlydefaults``, ``annotations``). diff --git a/Lib/inspect.py b/Lib/inspect.py index 1e94a9c..a089be6 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -16,7 +16,7 @@ Here are some of the useful functions provided by this module: getmodule() - determine the module that an object came from getclasstree() - arrange classes so as to represent their hierarchy - getargspec(), getargvalues(), getcallargs() - get info about function arguments + getargvalues(), getcallargs() - get info about function arguments getfullargspec() - same, with support for Python 3 features formatargspec(), formatargvalues() - format an argument spec getouterframes(), getinnerframes() - get info about frames @@ -1018,8 +1018,6 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping argument names to annotations. - The first four items in the tuple correspond to getargspec(). - This function is deprecated, use inspect.signature() instead. """ @@ -1130,8 +1128,7 @@ def formatargspec(args, varargs=None, varkw=None, defaults=None, formatvalue=lambda value: '=' + repr(value), formatreturns=lambda text: ' -> ' + text, formatannotation=formatannotation): - """Format an argument spec from the values returned by getargspec - or getfullargspec. + """Format an argument spec from the values returned by getfullargspec. The first seven arguments are (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). The other five arguments diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 309ee2a..db15b39 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -38,7 +38,7 @@ from test.test_import import _ready_to_import # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, # isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, # getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, -# getclasstree, getargspec, getargvalues, formatargspec, formatargvalues, +# getclasstree, getargvalues, formatargspec, formatargvalues, # currentframe, stack, trace, isdatadescriptor # NOTE: There are some additional tests relating to interaction with -- cgit v0.12 From daffc916aabb8424b6708039e64054f792fb660d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 31 Jul 2015 07:58:56 -0700 Subject: Issue #24681: Move the most likely test first in set_add_entry(). --- Objects/setobject.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 0e443c4..c590bbf 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -171,12 +171,15 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); - if (cmp < 0) /* unlikely */ - goto comparison_error; - if (table != so->table || entry->key != startkey) /* unlikely */ - goto restart; if (cmp > 0) /* likely */ goto found_active; + if (cmp < 0) + goto comparison_error; + /* Continuing the search from the current entry only makes + sense if the table and entry are unchanged; otherwise, + we have to restart from the beginning */ + if (table != so->table || entry->key != startkey) + goto restart; mask = so->mask; /* help avoid a register spill */ } else if (entry->hash == -1 && freeslot == NULL) @@ -200,12 +203,12 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Py_INCREF(startkey); cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); Py_DECREF(startkey); + if (cmp > 0) + goto found_active; if (cmp < 0) goto comparison_error; if (table != so->table || entry->key != startkey) goto restart; - if (cmp > 0) - goto found_active; mask = so->mask; } else if (entry->hash == -1 && freeslot == NULL) -- cgit v0.12 From 12f896c52128178b8b220ba63084cd68a4c15574 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 31 Jul 2015 12:03:20 -0700 Subject: Fix minor typo in a comment --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d2c0448..749cf6b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1047,7 +1047,7 @@ deque_clear(dequeobject *deque) static int valid_index(Py_ssize_t i, Py_ssize_t limit) { - /* The cast to size_t let us use just a single comparison + /* The cast to size_t lets us use just a single comparison to check whether i is in the range: 0 <= i < limit */ return (size_t) i < (size_t) limit; } -- cgit v0.12 From fbffdef47dd711d2441ff415f00c72ae7102e80f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 09:53:00 -0700 Subject: Issue #24762: Speed-up frozenset_hash() and greatly beef-up the comments. --- Objects/setobject.c | 70 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index c590bbf..2d4d8cd 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -739,41 +739,57 @@ set_traverse(PySetObject *so, visitproc visit, void *arg) return 0; } -static Py_hash_t -frozenset_hash(PyObject *self) +/* Work to increase the bit dispersion for closely spaced hash values. + This is important because some use cases have many combinations of a + small number of elements with nearby hashes so that many distinct + combinations collapse to only a handful of distinct hash values. */ + +static Py_uhash_t +_shuffle_bits(Py_uhash_t h) { - /* Most of the constants in this hash algorithm are randomly choosen - large primes with "interesting bit patterns" and that passed - tests for good collision statistics on a variety of problematic - datasets such as: + return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; +} - ps = [] - for r in range(21): - ps += itertools.combinations(range(20), r) - num_distinct_hashes = len({hash(frozenset(s)) for s in ps}) +/* Most of the constants in this hash algorithm are randomly chosen + large primes with "interesting bit patterns" and that passed tests + for good collision statistics on a variety of problematic datasets + including powersets and graph structures (such as David Eppstein's + graph recipes in Lib/test/test_set.py) */ - */ +static Py_hash_t +frozenset_hash(PyObject *self) +{ PySetObject *so = (PySetObject *)self; - Py_uhash_t h, hash = 1927868237UL; + Py_uhash_t hash = 1927868237UL; setentry *entry; - Py_ssize_t pos = 0; - - if (so->hash != -1) - return so->hash; + /* Make hash(frozenset({0})) distinct from hash(frozenset()) */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; - while (set_next(so, &pos, &entry)) { - /* Work to increase the bit dispersion for closely spaced hash - values. This is important because some use cases have many - combinations of a small number of elements with nearby - hashes so that many distinct combinations collapse to only - a handful of distinct hash values. */ - h = entry->hash; - hash ^= ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; - } - /* Make the final result spread-out in a different pattern - than the algorithm for tuples or other python objects. */ + + /* Xor-in shuffled bits from every entry's hash field because xor is + commutative and a frozenset hash should be independent of order. + + For speed, include null entries and dummy entries and then + subtract out their effect afterwards so that the final hash + depends only on active entries. This allows the code to be + vectorized by the compiler and it saves the unpredictable + branches that would arise when trying to exclude null and dummy + entries on every iteration. */ + + for (entry = so->table; entry <= &so->table[so->mask]; entry++) + hash ^= _shuffle_bits(entry->hash); + + /* Remove the effect of an odd number NULL entries */ + if ((so->mask + 1 - so->fill) & 1) + hash ^= _shuffle_bits(0); + + /* Remove the effect of an odd number of dummy entries */ + if ((so->fill - so->used) & 1) + hash ^= _shuffle_bits(-1); + + /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; + if (hash == (Py_uhash_t)-1) hash = 590923713UL; so->hash = hash; -- cgit v0.12 From 36c0500990a383a47228855bae712595f7f5e1d5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 10:57:42 -0700 Subject: Tweak the comments --- Objects/setobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 2d4d8cd..d638c13 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -763,7 +763,7 @@ frozenset_hash(PyObject *self) Py_uhash_t hash = 1927868237UL; setentry *entry; - /* Make hash(frozenset({0})) distinct from hash(frozenset()) */ + /* Initial dispersion based on the number of active entries */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; /* Xor-in shuffled bits from every entry's hash field because xor is @@ -790,8 +790,10 @@ frozenset_hash(PyObject *self) /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; + /* -1 is reserved as an error code */ if (hash == (Py_uhash_t)-1) hash = 590923713UL; + so->hash = hash; return hash; } -- cgit v0.12 From a286a51ae1a94cd85f7aee0bb55e44a363761325 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 11:07:11 -0700 Subject: Fix comment typo --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index d638c13..24424ad 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -779,7 +779,7 @@ frozenset_hash(PyObject *self) for (entry = so->table; entry <= &so->table[so->mask]; entry++) hash ^= _shuffle_bits(entry->hash); - /* Remove the effect of an odd number NULL entries */ + /* Remove the effect of an odd number of NULL entries */ if ((so->mask + 1 - so->fill) & 1) hash ^= _shuffle_bits(0); -- cgit v0.12 From 9344bd828cd391c25dc5ee6b4699b0025791a8f8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 1 Aug 2015 15:21:41 -0700 Subject: Clarify comments on setentry invariants. --- Include/setobject.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Include/setobject.h b/Include/setobject.h index f17bc1b..87ec1c8 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -10,12 +10,13 @@ extern "C" { /* There are three kinds of entries in the table: -1. Unused: key == NULL -2. Active: key != NULL and key != dummy -3. Dummy: key == dummy +1. Unused: key == NULL and hash == 0 +2. Dummy: key == dummy and hash == -1 +3. Active: key != NULL and key != dummy and hash != -1 -The hash field of Unused slots have no meaning. -The hash field of Dummny slots are set to -1 +The hash field of Unused slots is always zero. + +The hash field of Dummy slots are set to -1 meaning that dummy entries can be detected by either entry->key==dummy or by entry->hash==-1. */ -- cgit v0.12 From b501a27ad8ec4b531a9c1057952d8f7c6ef8cb77 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 6 Aug 2015 22:15:22 -0700 Subject: Restore frozenset hash caching removed in cf707dd190a9 --- Objects/setobject.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/setobject.c b/Objects/setobject.c index 24424ad..0a065cc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -763,6 +763,9 @@ frozenset_hash(PyObject *self) Py_uhash_t hash = 1927868237UL; setentry *entry; + if (so->hash != -1) + return so->hash; + /* Initial dispersion based on the number of active entries */ hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; -- cgit v0.12 From 4148195c45862fb62df97943591e9771b5638156 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 7 Aug 2015 00:43:39 -0700 Subject: Move the active entry multiplication to later in the hash calculation --- Objects/setobject.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 0a065cc..03bb230 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -760,15 +760,12 @@ static Py_hash_t frozenset_hash(PyObject *self) { PySetObject *so = (PySetObject *)self; - Py_uhash_t hash = 1927868237UL; + Py_uhash_t hash = 0; setentry *entry; if (so->hash != -1) return so->hash; - /* Initial dispersion based on the number of active entries */ - hash *= (Py_uhash_t)PySet_GET_SIZE(self) + 1; - /* Xor-in shuffled bits from every entry's hash field because xor is commutative and a frozenset hash should be independent of order. @@ -790,6 +787,9 @@ frozenset_hash(PyObject *self) if ((so->fill - so->used) & 1) hash ^= _shuffle_bits(-1); + /* Factor in the number of active entries */ + hash ^= ((Py_uhash_t)PySet_GET_SIZE(self) + 1) * 1927868237UL; + /* Disperse patterns arising in nested frozensets */ hash = hash * 69069U + 907133923UL; -- cgit v0.12 From 577d2069da6d73fa8ce86613fa7088d5e2a9229f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 7 Aug 2015 18:50:24 -0400 Subject: Fix typo in comment. --- Objects/abstract.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 31cc0a8..039a4ca 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2131,7 +2131,7 @@ PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) /* PyObject_Call() must not be called with an exception set, because it may clear it (directly or indirectly) and so the - caller looses its exception */ + caller loses its exception */ assert(!PyErr_Occurred()); call = func->ob_type->tp_call; -- cgit v0.12 From 455b5092a1b60cc6d14c4d48729ec5c16e274df1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 9 Aug 2015 00:35:00 -0700 Subject: Add more tests of hash effectiveness. --- Lib/test/test_set.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 54de508..ade39fb 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -10,6 +10,8 @@ import sys import warnings import collections import collections.abc +import itertools +import string class PassThru(Exception): pass @@ -711,6 +713,28 @@ class TestFrozenSet(TestJointOps, unittest.TestCase): addhashvalue(hash(frozenset([e for e, m in elemmasks if m&i]))) self.assertEqual(len(hashvalues), 2**n) + def letter_range(n): + return string.ascii_letters[:n] + + def zf_range(n): + # https://en.wikipedia.org/wiki/Set-theoretic_definition_of_natural_numbers + nums = [frozenset()] + for i in range(n-1): + num = frozenset(nums) + nums.append(num) + return nums[:n] + + def powerset(s): + for i in range(len(s)+1): + yield from map(frozenset, itertools.combinations(s, i)) + + for n in range(18): + t = 2 ** n + mask = t - 1 + for nums in (range, letter_range, zf_range): + u = len({h & mask for h in map(hash, powerset(nums(n)))}) + self.assertGreater(4*u, t) + class FrozenSetSubclass(frozenset): pass -- cgit v0.12 From 57fbec8c7c6fa2de7e23161aa1c41764a6290925 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Aug 2015 13:33:51 +0300 Subject: Added section for 3.5.0rc1 in Misc/NEWS and moved relevant entities to it. --- Misc/NEWS | 77 ++++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8ec1f01..3c358b2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,50 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +Library +------- + +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. + +- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. + +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + +- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. + +- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. + Patch from Nicola Palumbo and Laurent De Buyst. + +- Issue #24426: Fast searching optimization in regular expressions now works + for patterns that starts with capturing groups. Fast searching optimization + now can't be disabled at compile time. + +- Issue #23661: unittest.mock side_effects can now be exceptions again. This + was a regression vs Python 3.4. Patch from Ignacio Rossi + +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. + +Documentation +------------- + +Tests +----- + + +What's New in Python 3.5.0 release candidate 1? +=============================================== + +Release date: 2015-08-09 + +Core and Builtins +----------------- + - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library @@ -18,6 +62,10 @@ Library - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. +- Issue #24634: Importing uuid should not try to load libc on Windows + +- Issue #24798: _msvccompiler.py doesn't properly support manifests + - Issue #4395: Better testing and documentation of binary operators. Patch by Martin Panter. @@ -38,41 +86,20 @@ Library - Issue #23779: imaplib raises TypeError if authenticator tries to abort. Patch from Craig Holmquist. -- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid - identifiers. Patch by Matthias Bussonnier. - - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. -- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. - - Issue #23254: Document how to close the TCPServer listening socket. Patch from Martin Panter. -- Issue #23426: run_setup was broken in distutils. - Patch from Alexander Belopolsky. +- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. -- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. +- Issue #23812: Fix asyncio.Queue.get() to avoid loosing items on cancellation. + Patch by Gustavo J. A. M. Carneiro. -- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can - only be used with 'r'. Patch from Jeff Balogh and John O'Connor. - -- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #24426: Fast searching optimization in regular expressions now works - for patterns that starts with capturing groups. Fast searching optimization - now can't be disabled at compile time. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. +- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'. Documentation ------------- -- cgit v0.12 From 846a1487cb2dea74dcb858b35fa406ac38641fe6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 9 Aug 2015 13:44:04 +0300 Subject: Backed out changeset 42e2e67b8e6f --- Misc/NEWS | 77 +++++++++++++++++++++------------------------------------------ 1 file changed, 25 insertions(+), 52 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 3c358b2..8ec1f01 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,50 +10,6 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- -Library -------- - -- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid - identifiers. Patch by Matthias Bussonnier. - -- Issue #23426: run_setup was broken in distutils. - Patch from Alexander Belopolsky. - -- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. - -- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can - only be used with 'r'. Patch from Jeff Balogh and John O'Connor. - -- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. - -- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. - Patch from Nicola Palumbo and Laurent De Buyst. - -- Issue #24426: Fast searching optimization in regular expressions now works - for patterns that starts with capturing groups. Fast searching optimization - now can't be disabled at compile time. - -- Issue #23661: unittest.mock side_effects can now be exceptions again. This - was a regression vs Python 3.4. Patch from Ignacio Rossi - -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. - -Documentation -------------- - -Tests ------ - - -What's New in Python 3.5.0 release candidate 1? -=============================================== - -Release date: 2015-08-09 - -Core and Builtins ------------------ - - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library @@ -62,10 +18,6 @@ Library - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. -- Issue #24634: Importing uuid should not try to load libc on Windows - -- Issue #24798: _msvccompiler.py doesn't properly support manifests - - Issue #4395: Better testing and documentation of binary operators. Patch by Martin Panter. @@ -86,20 +38,41 @@ Library - Issue #23779: imaplib raises TypeError if authenticator tries to abort. Patch from Craig Holmquist. +- Issue #24360: Improve __repr__ of argparse.Namespace() for invalid + identifiers. Patch by Matthias Bussonnier. + - Issue #23319: Fix ctypes.BigEndianStructure, swap correctly bytes. Patch written by Matthieu Gautier. +- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. + - Issue #23254: Document how to close the TCPServer listening socket. Patch from Martin Panter. -- Issue #19450: Update Windows and OS X installer builds to use SQLite 3.8.11. +- Issue #23426: run_setup was broken in distutils. + Patch from Alexander Belopolsky. - Issue #17527: Add PATCH to wsgiref.validator. Patch from Luca Sbardella. -- Issue #23812: Fix asyncio.Queue.get() to avoid loosing items on cancellation. - Patch by Gustavo J. A. M. Carneiro. +- Issue #13938: 2to3 converts StringTypes to a tuple. Patch from Mark Hammond. -- Issue #24791: Fix grammar regression for call syntax: 'g(*a or b)'. +- Issue #2091: open() accepted a 'U' mode string containing '+', but 'U' can + only be used with 'r'. Patch from Jeff Balogh and John O'Connor. + +- Issue #8585: improved tests for zipimporter2. Patch from Mark Lawrence. + +- Issue #18622: unittest.mock.mock_open().reset_mock would recurse infinitely. + Patch from Nicola Palumbo and Laurent De Buyst. + +- Issue #24426: Fast searching optimization in regular expressions now works + for patterns that starts with capturing groups. Fast searching optimization + now can't be disabled at compile time. + +- Issue #23661: unittest.mock side_effects can now be exceptions again. This + was a regression vs Python 3.4. Patch from Ignacio Rossi + +- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo + functions. Documentation ------------- -- cgit v0.12 From dfa95c9a8f2772d1e8e54aa5aa14c91d4971964f Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 10 Aug 2015 09:53:30 +1200 Subject: Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. --- Doc/library/urllib.parse.rst | 18 ++++++++++++++---- Doc/whatsnew/3.6.rst | 5 ++++- Lib/test/test_urlparse.py | 36 +++++++++++++++++------------------- Lib/urllib/parse.py | 3 +-- Misc/NEWS | 3 +++ 5 files changed, 39 insertions(+), 26 deletions(-) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 40098d0..7c075ad 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -115,8 +115,9 @@ or on combining URL components into a URL string. | | | if present | | +------------------+-------+--------------------------+----------------------+ - See section :ref:`urlparse-result-object` for more information on the result - object. + Reading the :attr:`port` attribute will raise a :exc:`ValueError` if + an invalid port is specified in the URL. See section + :ref:`urlparse-result-object` for more information on the result object. .. versionchanged:: 3.2 Added IPv6 URL parsing capabilities. @@ -126,6 +127,10 @@ or on combining URL components into a URL string. false), in accordance with :rfc:`3986`. Previously, a whitelist of schemes that support fragments existed. + .. versionchanged:: 3.6 + Out-of-range port numbers now raise :exc:`ValueError`, instead of + returning :const:`None`. + .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace') @@ -228,8 +233,13 @@ or on combining URL components into a URL string. | | | if present | | +------------------+-------+-------------------------+----------------------+ - See section :ref:`urlparse-result-object` for more information on the result - object. + Reading the :attr:`port` attribute will raise a :exc:`ValueError` if + an invalid port is specified in the URL. See section + :ref:`urlparse-result-object` for more information on the result object. + + .. versionchanged:: 3.6 + Out-of-range port numbers now raise :exc:`ValueError`, instead of + returning :const:`None`. .. function:: urlunsplit(parts) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 4e686ad..e2aa1e2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -162,7 +162,10 @@ that may require changes to your code. Changes in the Python API ------------------------- -* None yet. +* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of + :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results + now raises :exc:`ValueError` for out-of-range values, rather than + returning :const:`None`. See :issue:`20059`. Changes in the C API diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 0552f90..fcf5082 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -554,29 +554,27 @@ class UrlParseTestCase(unittest.TestCase): self.assertEqual(p.port, 80) self.assertEqual(p.geturl(), url) - # Verify an illegal port is returned as None + # Verify an illegal port raises ValueError url = b"HTTP://WWW.PYTHON.ORG:65536/doc/#frag" p = urllib.parse.urlsplit(url) - self.assertEqual(p.port, None) + with self.assertRaisesRegex(ValueError, "out of range"): + p.port def test_attributes_bad_port(self): - """Check handling of non-integer ports.""" - p = urllib.parse.urlsplit("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse("http://www.example.net:foo") - self.assertEqual(p.netloc, "www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - # Once again, repeat ourselves to test bytes - p = urllib.parse.urlsplit(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) - - p = urllib.parse.urlparse(b"http://www.example.net:foo") - self.assertEqual(p.netloc, b"www.example.net:foo") - self.assertRaises(ValueError, lambda: p.port) + """Check handling of invalid ports.""" + for bytes in (False, True): + for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): + for port in ("foo", "1.5", "-1", "0x10"): + with self.subTest(bytes=bytes, parse=parse, port=port): + netloc = "www.example.net:" + port + url = "http://" + netloc + if bytes: + netloc = netloc.encode("ascii") + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 01c9e58..5e2155c 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -156,9 +156,8 @@ class _NetlocResultMixinBase(object): port = self._hostinfo[1] if port is not None: port = int(port, 10) - # Return None on an illegal port if not ( 0 <= port <= 65535): - return None + raise ValueError("Port out of range 0-65535") return port diff --git a/Misc/NEWS b/Misc/NEWS index 8ec1f01..1aa6b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -15,6 +15,9 @@ Core and Builtins Library ------- +- Issue #20059: urllib.parse raises ValueError on all invalid ports. + Patch by Martin Panter. + - Issue #24824: Signatures of codecs.encode() and codecs.decode() now are compatible with pydoc. -- cgit v0.12 From df395991f6341be037a5c6df9b14d924d5ac9e95 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 12 Aug 2015 08:00:06 +1200 Subject: Issue #9232: Support trailing commas in function declarations. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. --- Doc/reference/compound_stmts.rst | 8 +-- Grammar/Grammar | 17 ++++--- Lib/test/test_grammar.py | 38 ++++++++++++++ Misc/NEWS | 4 ++ Python/ast.c | 23 +++++---- Python/graminit.c | 106 ++++++++++++++++++++++----------------- 6 files changed, 131 insertions(+), 65 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 01cfd6d..00e6476 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -471,10 +471,10 @@ A function definition defines a user-defined function object (see section decorators: `decorator`+ decorator: "@" `dotted_name` ["(" [`parameter_list` [","]] ")"] NEWLINE dotted_name: `identifier` ("." `identifier`)* - parameter_list: (`defparameter` ",")* - : | "*" [`parameter`] ("," `defparameter`)* ["," "**" `parameter`] - : | "**" `parameter` - : | `defparameter` [","] ) + parameter_list: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] + : | `parameter_list_starargs` + parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] + : | "**" `parameter` [","] parameter: `identifier` [":" `expression`] defparameter: `parameter` ["=" `expression`] funcname: `identifier` diff --git a/Grammar/Grammar b/Grammar/Grammar index 99fcea0..14bdecd 100644 --- a/Grammar/Grammar +++ b/Grammar/Grammar @@ -27,13 +27,18 @@ async_funcdef: ASYNC funcdef funcdef: 'def' NAME parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' - ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) tfpdef: NAME [':' test] -varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' - ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] - | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] +) vfpdef: NAME stmt: simple_stmt | compound_stmt diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index ec3d783..8f8d71c 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -295,6 +295,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) + self.assertRaises(SyntaxError, eval, "def f(*): pass") + self.assertRaises(SyntaxError, eval, "def f(*,): pass") + self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # keyword arguments after *arglist def f(*args, **kwargs): return args, kwargs @@ -352,6 +356,23 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") + # Check trailing commas are permitted in funcdef argument list + def f(a,): pass + def f(*args,): pass + def f(**kwds,): pass + def f(a, *args,): pass + def f(a, **kwds,): pass + def f(*args, b,): pass + def f(*, b,): pass + def f(*args, **kwds,): pass + def f(a, *args, b,): pass + def f(a, *, b,): pass + def f(a, *args, **kwds,): pass + def f(*args, b, **kwds,): pass + def f(*, b, **kwds,): pass + def f(a, *args, b, **kwds,): pass + def f(a, *, b, **kwds,): pass + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 @@ -370,6 +391,23 @@ class GrammarTests(unittest.TestCase): self.assertEqual(l6(1,2), 1+2+20) self.assertEqual(l6(1,2,k=10), 1+2+10) + # check that trailing commas are permitted + l10 = lambda a,: 0 + l11 = lambda *args,: 0 + l12 = lambda **kwds,: 0 + l13 = lambda a, *args,: 0 + l14 = lambda a, **kwds,: 0 + l15 = lambda *args, b,: 0 + l16 = lambda *, b,: 0 + l17 = lambda *args, **kwds,: 0 + l18 = lambda a, *args, b,: 0 + l19 = lambda a, *, b,: 0 + l20 = lambda a, *args, **kwds,: 0 + l21 = lambda *args, b, **kwds,: 0 + l22 = lambda *, b, **kwds,: 0 + l23 = lambda a, *args, b, **kwds,: 0 + l24 = lambda a, *, b, **kwds,: 0 + ### stmt: simple_stmt | compound_stmt # Tested below diff --git a/Misc/NEWS b/Misc/NEWS index 1aa6b09..c2ea6d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #9232: Modify Python's grammar to allow trailing commas in the + argument list of a function declaration. For example, "def f(*, a = + 3,): pass" is now legal. Patch from Mark Dickinson. + - Issue #24667: Resize odict in all cases that the underlying dict resizes. Library diff --git a/Python/ast.c b/Python/ast.c index b572088..c1c3907 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1260,16 +1260,20 @@ ast_for_arguments(struct compiling *c, const node *n) and varargslist (lambda definition). parameters: '(' [typedargslist] ')' - typedargslist: ((tfpdef ['=' test] ',')* - ('*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] - | '**' tfpdef) - | tfpdef ['=' test] (',' tfpdef ['=' test])* [',']) + typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [ + '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]] + | '**' tfpdef [',']) tfpdef: NAME [':' test] - varargslist: ((vfpdef ['=' test] ',')* - ('*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] - | '**' vfpdef) - | vfpdef ['=' test] (',' vfpdef ['=' test])* [',']) + varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [ + '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [',']]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]] + | '**' vfpdef [','] + ) vfpdef: NAME + */ int i, j, k, nposargs = 0, nkwonlyargs = 0; int nposdefaults = 0, found_default = 0; @@ -1371,7 +1375,8 @@ ast_for_arguments(struct compiling *c, const node *n) i += 2; /* the name and the comma */ break; case STAR: - if (i+1 >= NCH(n)) { + if (i+1 >= NCH(n) || + (i+2 == NCH(n) && TYPE(CHILD(n, i+1)) == COMMA)) { ast_error(c, CHILD(n, i), "named arguments must follow bare *"); return NULL; diff --git a/Python/graminit.c b/Python/graminit.c index 8212b2a..354dc12 100644 --- a/Python/graminit.c +++ b/Python/graminit.c @@ -204,11 +204,13 @@ static arc arcs_9_6[2] = { {32, 7}, {0, 6}, }; -static arc arcs_9_7[2] = { +static arc arcs_9_7[3] = { {30, 12}, {34, 3}, + {0, 7}, }; -static arc arcs_9_8[1] = { +static arc arcs_9_8[2] = { + {32, 13}, {0, 8}, }; static arc arcs_9_9[2] = { @@ -221,35 +223,39 @@ static arc arcs_9_10[3] = { {0, 10}, }; static arc arcs_9_11[3] = { - {30, 13}, - {32, 14}, + {30, 14}, + {32, 15}, {0, 11}, }; static arc arcs_9_12[3] = { {32, 7}, - {31, 15}, + {31, 16}, {0, 12}, }; -static arc arcs_9_13[2] = { - {32, 14}, +static arc arcs_9_13[1] = { {0, 13}, }; static arc arcs_9_14[2] = { - {30, 16}, + {32, 15}, + {0, 14}, +}; +static arc arcs_9_15[3] = { + {30, 17}, {34, 3}, + {0, 15}, }; -static arc arcs_9_15[1] = { +static arc arcs_9_16[1] = { {26, 6}, }; -static arc arcs_9_16[3] = { - {32, 14}, - {31, 17}, - {0, 16}, +static arc arcs_9_17[3] = { + {32, 15}, + {31, 18}, + {0, 17}, }; -static arc arcs_9_17[1] = { - {26, 13}, +static arc arcs_9_18[1] = { + {26, 14}, }; -static state states_9[18] = { +static state states_9[19] = { {3, arcs_9_0}, {3, arcs_9_1}, {3, arcs_9_2}, @@ -257,17 +263,18 @@ static state states_9[18] = { {1, arcs_9_4}, {4, arcs_9_5}, {2, arcs_9_6}, - {2, arcs_9_7}, - {1, arcs_9_8}, + {3, arcs_9_7}, + {2, arcs_9_8}, {2, arcs_9_9}, {3, arcs_9_10}, {3, arcs_9_11}, {3, arcs_9_12}, - {2, arcs_9_13}, + {1, arcs_9_13}, {2, arcs_9_14}, - {1, arcs_9_15}, - {3, arcs_9_16}, - {1, arcs_9_17}, + {3, arcs_9_15}, + {1, arcs_9_16}, + {3, arcs_9_17}, + {1, arcs_9_18}, }; static arc arcs_10_0[1] = { {23, 1}, @@ -319,11 +326,13 @@ static arc arcs_11_6[2] = { {32, 7}, {0, 6}, }; -static arc arcs_11_7[2] = { +static arc arcs_11_7[3] = { {36, 12}, {34, 3}, + {0, 7}, }; -static arc arcs_11_8[1] = { +static arc arcs_11_8[2] = { + {32, 13}, {0, 8}, }; static arc arcs_11_9[2] = { @@ -336,35 +345,39 @@ static arc arcs_11_10[3] = { {0, 10}, }; static arc arcs_11_11[3] = { - {36, 13}, - {32, 14}, + {36, 14}, + {32, 15}, {0, 11}, }; static arc arcs_11_12[3] = { {32, 7}, - {31, 15}, + {31, 16}, {0, 12}, }; -static arc arcs_11_13[2] = { - {32, 14}, +static arc arcs_11_13[1] = { {0, 13}, }; static arc arcs_11_14[2] = { - {36, 16}, + {32, 15}, + {0, 14}, +}; +static arc arcs_11_15[3] = { + {36, 17}, {34, 3}, + {0, 15}, }; -static arc arcs_11_15[1] = { +static arc arcs_11_16[1] = { {26, 6}, }; -static arc arcs_11_16[3] = { - {32, 14}, - {31, 17}, - {0, 16}, +static arc arcs_11_17[3] = { + {32, 15}, + {31, 18}, + {0, 17}, }; -static arc arcs_11_17[1] = { - {26, 13}, +static arc arcs_11_18[1] = { + {26, 14}, }; -static state states_11[18] = { +static state states_11[19] = { {3, arcs_11_0}, {3, arcs_11_1}, {3, arcs_11_2}, @@ -372,17 +385,18 @@ static state states_11[18] = { {1, arcs_11_4}, {4, arcs_11_5}, {2, arcs_11_6}, - {2, arcs_11_7}, - {1, arcs_11_8}, + {3, arcs_11_7}, + {2, arcs_11_8}, {2, arcs_11_9}, {3, arcs_11_10}, {3, arcs_11_11}, {3, arcs_11_12}, - {2, arcs_11_13}, + {1, arcs_11_13}, {2, arcs_11_14}, - {1, arcs_11_15}, - {3, arcs_11_16}, - {1, arcs_11_17}, + {3, arcs_11_15}, + {1, arcs_11_16}, + {3, arcs_11_17}, + {1, arcs_11_18}, }; static arc arcs_12_0[1] = { {23, 1}, @@ -1879,11 +1893,11 @@ static dfa dfas[85] = { "\000\000\100\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {264, "parameters", 0, 4, states_8, "\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {265, "typedargslist", 0, 18, states_9, + {265, "typedargslist", 0, 19, states_9, "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {266, "tfpdef", 0, 4, states_10, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {267, "varargslist", 0, 18, states_11, + {267, "varargslist", 0, 19, states_11, "\000\000\200\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, {268, "vfpdef", 0, 2, states_12, "\000\000\200\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, -- cgit v0.12 From 80d62e628b25502239b5a3dc7c5aaca3deaa7e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 13 Aug 2015 20:37:08 +0100 Subject: Issue #23530: fix clinic comment. --- Modules/clinic/posixmodule.c.h | 10 +++++++--- Modules/posixmodule.c | 12 +++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 98eeae4..a7045a8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2116,7 +2116,7 @@ PyDoc_STRVAR(os_sched_getaffinity__doc__, "sched_getaffinity($module, pid, /)\n" "--\n" "\n" -"Return the affinity of the process identified by pid.\n" +"Return the affinity of the process identified by pid (or the current process if zero).\n" "\n" "The affinity is returned as a set of CPU identifiers."); @@ -5178,7 +5178,11 @@ PyDoc_STRVAR(os_cpu_count__doc__, "cpu_count($module, /)\n" "--\n" "\n" -"Return the number of CPUs in the system; return None if indeterminable."); +"Return the number of CPUs in the system; return None if indeterminable.\n" +"\n" +"This number is not equivalent to the number of CPUs the current process can\n" +"use. The number of usable CPUs can be obtained with\n" +"``len(os.sched_getaffinity(0))``"); #define OS_CPU_COUNT_METHODDEF \ {"cpu_count", (PyCFunction)os_cpu_count, METH_NOARGS, os_cpu_count__doc__}, @@ -5788,4 +5792,4 @@ exit: #ifndef OS_SET_HANDLE_INHERITABLE_METHODDEF #define OS_SET_HANDLE_INHERITABLE_METHODDEF #endif /* !defined(OS_SET_HANDLE_INHERITABLE_METHODDEF) */ -/*[clinic end generated code: output=f3f92b2d2e2c3fe3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=35b50461dbecd65d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 23d74a3..4d25ed1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5767,7 +5767,7 @@ The affinity is returned as a set of CPU identifiers. static PyObject * os_sched_getaffinity_impl(PyModuleDef *module, pid_t pid) -/*[clinic end generated code: output=b431a8f310e369e7 input=eaf161936874b8a1]*/ +/*[clinic end generated code: output=b431a8f310e369e7 input=983ce7cb4a565980]*/ { int cpu, ncpus, count; size_t setsize; @@ -11201,14 +11201,16 @@ get_terminal_size(PyObject *self, PyObject *args) /*[clinic input] os.cpu_count -Return the number of CPUs in the system; return None if indeterminable. This -number is not equivalent to the number of CPUs the current process can use. -The number of usable CPUs can be obtained with ``len(os.sched_getaffinity(0))`` +Return the number of CPUs in the system; return None if indeterminable. + +This number is not equivalent to the number of CPUs the current process can +use. The number of usable CPUs can be obtained with +``len(os.sched_getaffinity(0))`` [clinic start generated code]*/ static PyObject * os_cpu_count_impl(PyModuleDef *module) -/*[clinic end generated code: output=c59ee7f6bce832b8 input=d55e2f8f3823a628]*/ +/*[clinic end generated code: output=c59ee7f6bce832b8 input=e7c8f4ba6dbbadd3]*/ { int ncpu = 0; #ifdef MS_WINDOWS -- cgit v0.12 From 7126eaa2d1185065a6d43d90185a6a56c7d5df18 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Fri, 14 Aug 2015 11:50:14 +1200 Subject: Issue #21159: Improve message in configparser.InterpolationMissingOptionError. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch from Åukasz Langa. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index a8bc453..9e823c0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,9 @@ Core and Builtins Library ------- +- Issue #21159: Improve message in configparser.InterpolationMissingOptionError. + Patch from Åukasz Langa. + - Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. -- cgit v0.12 From 28c995d03b6e2f5a77c42a97e95368767846f14f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Aug 2015 02:07:41 -0700 Subject: Minor cleanup. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 749cf6b..35034a5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2167,13 +2167,13 @@ _count_elements(PyObject *self, PyObject *args) oldval = _PyDict_GetItem_KnownHash(mapping, key, hash); if (oldval == NULL) { - if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) == -1) + if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) < 0) goto done; } else { newval = PyNumber_Add(oldval, one); if (newval == NULL) goto done; - if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) == -1) + if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) < 0) goto done; Py_CLEAR(newval); } @@ -2199,7 +2199,7 @@ _count_elements(PyObject *self, PyObject *args) Py_DECREF(oldval); if (newval == NULL) break; - if (PyObject_SetItem(mapping, key, newval) == -1) + if (PyObject_SetItem(mapping, key, newval) < 0) break; Py_CLEAR(newval); Py_DECREF(key); -- cgit v0.12 From 721b9aab10a4c4f95c3fba08af24337efe63384c Mon Sep 17 00:00:00 2001 From: Larry Hastings Date: Fri, 14 Aug 2015 08:03:30 -0700 Subject: One-character typo fix. os.path.commonprefix always returns a valid *path*, not a valid *patch*. --- Doc/whatsnew/3.5.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index a974130..11f871b 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -729,7 +729,7 @@ os.path * New :func:`~os.path.commonpath` function that extracts common path prefix. Unlike the :func:`~os.path.commonprefix` function, it always returns a valid - patch. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) + path. (Contributed by Rafik Draoui and Serhiy Storchaka in :issue:`10395`.) pickle ------ -- cgit v0.12 From b468e1f59533fd296df7d899e9a71650871f30ce Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Aug 2015 14:10:49 -0700 Subject: Neaten-up vertical alignment of comments --- Modules/itertoolsmodule.c | 124 +++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ac31e2f..349da0f 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -178,7 +178,7 @@ static PyMethodDef groupby_methods[] = { reduce_doc}, {"__setstate__", (PyCFunction)groupby_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; PyDoc_STRVAR(groupby_doc, @@ -364,7 +364,7 @@ static PyTypeObject _grouper_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ - (traverseproc)_grouper_traverse,/* tp_traverse */ + (traverseproc)_grouper_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -582,7 +582,7 @@ static PyMethodDef teedataobject_methods[] = { PyDoc_STRVAR(teedataobject_doc, "Data container common to multiple tee objects."); static PyTypeObject teedataobject_type = { - PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ + PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ "itertools._tee_dataobject", /* tp_name */ sizeof(teedataobject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -602,7 +602,7 @@ static PyTypeObject teedataobject_type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ teedataobject_doc, /* tp_doc */ (traverseproc)teedataobject_traverse, /* tp_traverse */ (inquiry)teedataobject_clear, /* tp_clear */ @@ -791,7 +791,7 @@ static PyTypeObject tee_type = { (traverseproc)tee_traverse, /* tp_traverse */ (inquiry)tee_clear, /* tp_clear */ 0, /* tp_richcompare */ - offsetof(teeobject, weakreflist), /* tp_weaklistoffset */ + offsetof(teeobject, weakreflist), /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)tee_next, /* tp_iternext */ tee_methods, /* tp_methods */ @@ -1189,13 +1189,13 @@ static PyTypeObject dropwhile_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ dropwhile_doc, /* tp_doc */ - (traverseproc)dropwhile_traverse, /* tp_traverse */ + (traverseproc)dropwhile_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)dropwhile_next, /* tp_iternext */ - dropwhile_methods, /* tp_methods */ + dropwhile_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -1353,7 +1353,7 @@ static PyTypeObject takewhile_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ takewhile_doc, /* tp_doc */ - (traverseproc)takewhile_traverse, /* tp_traverse */ + (traverseproc)takewhile_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -2287,7 +2287,7 @@ product((0,1), (0,1), (0,1)) --> (0,0,0) (0,0,1) (0,1,0) (0,1,1) (1,0,0) ..."); static PyTypeObject product_type = { PyVarObject_HEAD_INIT(NULL, 0) "itertools.product", /* tp_name */ - sizeof(productobject), /* tp_basicsize */ + sizeof(productobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)product_dealloc, /* tp_dealloc */ @@ -2334,8 +2334,8 @@ static PyTypeObject product_type = { typedef struct { PyObject_HEAD PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ Py_ssize_t r; /* size of result tuple */ int stopped; /* set to 1 when the combinations iterator is exhausted */ } combinationsobject; @@ -2675,8 +2675,8 @@ static PyTypeObject combinations_type = { typedef struct { PyObject_HEAD PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ Py_ssize_t r; /* size of result tuple */ int stopped; /* set to 1 when the cwr iterator is exhausted */ } cwrobject; @@ -3326,11 +3326,11 @@ permutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1)"); static PyTypeObject permutations_type = { PyVarObject_HEAD_INIT(NULL, 0) - "itertools.permutations", /* tp_name */ + "itertools.permutations", /* tp_name */ sizeof(permutationsobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)permutations_dealloc, /* tp_dealloc */ + (destructor)permutations_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3347,13 +3347,13 @@ static PyTypeObject permutations_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - permutations_doc, /* tp_doc */ - (traverseproc)permutations_traverse, /* tp_traverse */ + permutations_doc, /* tp_doc */ + (traverseproc)permutations_traverse,/* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ - (iternextfunc)permutations_next, /* tp_iternext */ + (iternextfunc)permutations_next, /* tp_iternext */ permuations_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ @@ -3364,7 +3364,7 @@ static PyTypeObject permutations_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - permutations_new, /* tp_new */ + permutations_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; @@ -3664,44 +3664,44 @@ static PyTypeObject compress_type = { PyVarObject_HEAD_INIT(NULL, 0) "itertools.compress", /* tp_name */ sizeof(compressobject), /* tp_basicsize */ - 0, /* tp_itemsize */ + 0, /* tp_itemsize */ /* methods */ (destructor)compress_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - compress_doc, /* tp_doc */ - (traverseproc)compress_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ + Py_TPFLAGS_BASETYPE, /* tp_flags */ + compress_doc, /* tp_doc */ + (traverseproc)compress_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ (iternextfunc)compress_next, /* tp_iternext */ - compress_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - compress_new, /* tp_new */ - PyObject_GC_Del, /* tp_free */ + compress_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + compress_new, /* tp_new */ + PyObject_GC_Del, /* tp_free */ }; @@ -3824,7 +3824,7 @@ static PyTypeObject filterfalse_type = { sizeof(filterfalseobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)filterfalse_dealloc, /* tp_dealloc */ + (destructor)filterfalse_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3842,7 +3842,7 @@ static PyTypeObject filterfalse_type = { Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ filterfalse_doc, /* tp_doc */ - (traverseproc)filterfalse_traverse, /* tp_traverse */ + (traverseproc)filterfalse_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -4081,15 +4081,15 @@ static PyTypeObject count_type = { 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_BASETYPE, /* tp_flags */ count_doc, /* tp_doc */ - (traverseproc)count_traverse, /* tp_traverse */ + (traverseproc)count_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)count_next, /* tp_iternext */ - count_methods, /* tp_methods */ + count_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ @@ -4488,7 +4488,7 @@ static PyTypeObject ziplongest_type = { sizeof(ziplongestobject), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)zip_longest_dealloc, /* tp_dealloc */ + (destructor)zip_longest_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -4505,8 +4505,8 @@ static PyTypeObject ziplongest_type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ - zip_longest_doc, /* tp_doc */ - (traverseproc)zip_longest_traverse, /* tp_traverse */ + zip_longest_doc, /* tp_doc */ + (traverseproc)zip_longest_traverse, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -4522,7 +4522,7 @@ static PyTypeObject ziplongest_type = { 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ - zip_longest_new, /* tp_new */ + zip_longest_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; -- cgit v0.12 From 79c878d5f24d58fe09e842f9a1cc13d6cf560aab Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 13:51:59 -0700 Subject: Fix crash in itertools.cycle.__setstate__() caused by lack of type checking. Will backport after the 3.6 release is done. --- Lib/test/test_itertools.py | 33 +++++++++++++++++++++++++++++++++ Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index fcd8869..53d6564 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,39 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + def test_cycle_setstate(self): + # Verify both modes for restoring state + + # Mode 0 is efficient. It uses an incompletely consumed input + # iterator to build a cycle object and then passes in state with + # a list of previously consumed values. There is no data + # overlap bewteen the two. + c = cycle('defg') + c.__setstate__((list('abc'), 0)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # Mode 1 is inefficient. It starts with a cycle object built + # from an iterator over the remaining elements in a partial + # cycle and then passes in state with all of the previously + # seen values (this overlaps values included in the iterator). + c = cycle('defg') + c.__setstate__((list('abcdefg'), 1)) + self.assertEqual(take(20, c), list('defgabcdefgabcdefgab')) + + # The first argument to setstate needs to be a tuple + with self.assertRaises(SystemError): + cycle('defg').__setstate__([list('abcdefg'), 0]) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + c = cycle('defg') + c.__setstate__((dict.fromkeys('defg'), 0)) + take(20, c) + + # The first argument in the setstate tuple must be a list + with self.assertRaises(TypeError): + cycle('defg').__setstate__((list('abcdefg'), 'x')) + def test_groupby(self): # Check whether it accepts arguments correctly self.assertEqual([], list(groupby([]))) diff --git a/Misc/NEWS b/Misc/NEWS index 574e279..a8b93e6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Library - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. Patch from Åukasz Langa. +- Fix crash in itertools.cycle.__setstate__() when the first argument wasn't + a list. + - Issue #20059: urllib.parse raises ValueError on all invalid ports. Patch by Martin Panter. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 349da0f..2cca6b3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -973,7 +973,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) { PyObject *saved=NULL; int firstpass; - if (!PyArg_ParseTuple(state, "Oi", &saved, &firstpass)) + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; Py_CLEAR(lz->saved); lz->saved = saved; -- cgit v0.12 From a166ce561c8c80dc892deab491ff5625cc0409c8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:45:49 -0700 Subject: Add more tests for pickling itertools.cycle --- Lib/test/test_itertools.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 53d6564..c012efd 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -613,6 +613,23 @@ class TestBasicOps(unittest.TestCase): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, cycle('abc')) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + # test with partial consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + + # test with completely consumed input iterable + it = iter('abcde') + c = cycle(it) + _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + p = pickle.dumps(c, proto) + d = pickle.loads(p) # rebuild the cycle object + self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_setstate(self): # Verify both modes for restoring state -- cgit v0.12 From 1cadf769b3d453777a116a1d11831ef625ebbe99 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 14:47:27 -0700 Subject: Improve comment --- Lib/test/test_itertools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index c012efd..81e1711 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -617,7 +617,7 @@ class TestBasicOps(unittest.TestCase): # test with partial consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(2)] # consume to 2 of 5 inputs + _ = [next(c) for i in range(2)] # consume 2 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) @@ -625,7 +625,7 @@ class TestBasicOps(unittest.TestCase): # test with completely consumed input iterable it = iter('abcde') c = cycle(it) - _ = [next(c) for i in range(7)] # consume to 7 of 5 inputs + _ = [next(c) for i in range(7)] # consume 7 of 5 inputs p = pickle.dumps(c, proto) d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) -- cgit v0.12 From 98958fedb6b8f6f8f9bc8587f7c9884ea7c47e69 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 15:09:30 -0700 Subject: Remove dead code (unreachable) --- Modules/itertoolsmodule.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2cca6b3..0d1a15b 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -940,12 +940,9 @@ cycle_next(cycleobject *lz) } return item; } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else + /* Note: StopIteration is already cleared by PyIter_Next() */ + if (PyErr_Occurred()) return NULL; - } if (PyList_Size(lz->saved) == 0) return NULL; it = PyObject_GetIter(lz->saved); -- cgit v0.12 From c39786dc08d68ebb832c3b249c42dcdaba38b3a9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 15 Aug 2015 15:16:12 -0700 Subject: Fix oddly placed whitespace --- Modules/itertoolsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 0d1a15b..18d55d8 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -963,7 +963,7 @@ cycle_reduce(cycleobject *lz) */ return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), lz->it, lz->saved, lz->firstpass); - } +} static PyObject * cycle_setstate(cycleobject *lz, PyObject *state) -- cgit v0.12 From 2a75e8f95899431f44a9af9e23ca504d3e73a2ad Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 08:32:01 -0700 Subject: Issue #24842: Cross-reference types.SimpleNamespace from the namedtuple docs --- Doc/library/collections.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index f2befe7..aeb1c52 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -962,6 +962,9 @@ and more efficient to use a simple :class:`~enum.Enum`: constructor that is convenient for use cases where named tuples are being subclassed. + * :meth:`types.SimpleNamespace` for a mutable namespace based on an underlying + dictionary instead of a tuple. + :class:`OrderedDict` objects ---------------------------- -- cgit v0.12 From b5244a3fe5aa15c41311fe124f5ba1b209f7c316 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:24:20 -0700 Subject: Inline PyIter_Next() matching what was done for other itertools. --- Modules/itertoolsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 18d55d8..bba0935 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1860,7 +1860,7 @@ chain_next(chainobject *lz) return NULL; /* input not iterable */ } } - item = PyIter_Next(lz->active); + item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); if (item != NULL) return item; if (PyErr_Occurred()) { @@ -3434,7 +3434,7 @@ accumulate_next(accumulateobject *lz) { PyObject *val, *oldtotal, *newtotal; - val = PyIter_Next(lz->it); + val = (*Py_TYPE(lz->it)->tp_iternext)(lz->it); if (val == NULL) return NULL; -- cgit v0.12 From a6a2d44dc7f50a75c3df742d9fc2c30217436c40 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:38:07 -0700 Subject: Neaten-up whitespace, vertical alignment, and line-wrapping. --- Modules/itertoolsmodule.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index bba0935..533e1d1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -759,9 +759,9 @@ PyDoc_STRVAR(teeobject_doc, "Iterator wrapped to make it copyable"); static PyMethodDef tee_methods[] = { - {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, - {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, - {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, + {"__copy__", (PyCFunction)tee_copy, METH_NOARGS, teecopy_doc}, + {"__reduce__", (PyCFunction)tee_reduce, METH_NOARGS, reduce_doc}, + {"__setstate__", (PyCFunction)tee_setstate, METH_O, setstate_doc}, {NULL, NULL} /* sentinel */ }; @@ -1407,7 +1407,8 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyErr_Occurred()) PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Stop argument for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } } @@ -1422,14 +1423,16 @@ islice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (PyErr_Occurred()) PyErr_Clear(); PyErr_SetString(PyExc_ValueError, - "Stop argument for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Stop argument for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } } } if (start<0 || stop<-1) { PyErr_SetString(PyExc_ValueError, - "Indices for islice() must be None or an integer: 0 <= x <= sys.maxsize."); + "Indices for islice() must be None or " + "an integer: 0 <= x <= sys.maxsize."); return NULL; } @@ -1845,19 +1848,19 @@ chain_next(chainobject *lz) PyObject *item; if (lz->source == NULL) - return NULL; /* already stopped */ + return NULL; /* already stopped */ if (lz->active == NULL) { PyObject *iterable = PyIter_Next(lz->source); if (iterable == NULL) { Py_CLEAR(lz->source); - return NULL; /* no more input sources */ + return NULL; /* no more input sources */ } lz->active = PyObject_GetIter(iterable); Py_DECREF(iterable); if (lz->active == NULL) { Py_CLEAR(lz->source); - return NULL; /* input not iterable */ + return NULL; /* input not iterable */ } } item = (*Py_TYPE(lz->active)->tp_iternext)(lz->active); @@ -1867,10 +1870,10 @@ chain_next(chainobject *lz) if (PyErr_ExceptionMatches(PyExc_StopIteration)) PyErr_Clear(); else - return NULL; /* input raised an exception */ + return NULL; /* input raised an exception */ } Py_CLEAR(lz->active); - return chain_next(lz); /* recurse and use next active */ + return chain_next(lz); /* recurse and use next active */ } static PyObject * @@ -1928,7 +1931,7 @@ static PyMethodDef chain_methods[] = { reduce_doc}, {"__setstate__", (PyCFunction)chain_setstate, METH_O, setstate_doc}, - {NULL, NULL} /* sentinel */ + {NULL, NULL} /* sentinel */ }; static PyTypeObject chain_type = { @@ -1980,7 +1983,7 @@ static PyTypeObject chain_type = { typedef struct { PyObject_HEAD - PyObject *pools; /* tuple of pool tuples */ + PyObject *pools; /* tuple of pool tuples */ Py_ssize_t *indices; /* one index per pool */ PyObject *result; /* most recently returned result tuple */ int stopped; /* set to 1 when the product iterator is exhausted */ @@ -3779,8 +3782,7 @@ filterfalse_next(filterfalseobject *lz) ok = PyObject_IsTrue(item); } else { PyObject *good; - good = PyObject_CallFunctionObjArgs(lz->func, - item, NULL); + good = PyObject_CallFunctionObjArgs(lz->func, item, NULL); if (good == NULL) { Py_DECREF(item); return NULL; -- cgit v0.12 From ca3788c2e8de9c60252d7a8c5fcf0e0bfa25333d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 14:49:24 -0700 Subject: Issue #24874: Speed-up itertools and make it pickles more compact. --- Misc/NEWS | 3 +++ Modules/itertoolsmodule.c | 62 +++++++++++++++++++++++++++++++---------------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index a8b93e6..a3a749d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -25,6 +25,9 @@ Library - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. Patch from Åukasz Langa. +- Issue #24874: Improve speed of itertools.cycle() and make its + pickle more compact. + - Fix crash in itertools.cycle.__setstate__() when the first argument wasn't a list. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 533e1d1..61ad0c8 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,4 +1,6 @@ +#define PY_SSIZE_T_CLEAN + #include "Python.h" #include "structmember.h" @@ -863,6 +865,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *saved; + Py_ssize_t index; int firstpass; } cycleobject; @@ -902,6 +905,7 @@ cycle_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } lz->it = it; lz->saved = saved; + lz->index = 0; lz->firstpass = 0; return (PyObject *)lz; @@ -911,15 +915,16 @@ static void cycle_dealloc(cycleobject *lz) { PyObject_GC_UnTrack(lz); - Py_XDECREF(lz->saved); Py_XDECREF(lz->it); + Py_XDECREF(lz->saved); Py_TYPE(lz)->tp_free(lz); } static int cycle_traverse(cycleobject *lz, visitproc visit, void *arg) { - Py_VISIT(lz->it); + if (lz->it) + Py_VISIT(lz->it); Py_VISIT(lz->saved); return 0; } @@ -928,13 +933,13 @@ static PyObject * cycle_next(cycleobject *lz) { PyObject *item; - PyObject *it; - PyObject *tmp; - while (1) { + if (lz->it != NULL) { item = PyIter_Next(lz->it); if (item != NULL) { - if (!lz->firstpass && PyList_Append(lz->saved, item)) { + if (lz->firstpass) + return item; + if (PyList_Append(lz->saved, item)) { Py_DECREF(item); return NULL; } @@ -942,27 +947,41 @@ cycle_next(cycleobject *lz) } /* Note: StopIteration is already cleared by PyIter_Next() */ if (PyErr_Occurred()) - return NULL; - if (PyList_Size(lz->saved) == 0) - return NULL; - it = PyObject_GetIter(lz->saved); - if (it == NULL) return NULL; - tmp = lz->it; - lz->it = it; - lz->firstpass = 1; - Py_DECREF(tmp); + Py_CLEAR(lz->it); } + if (Py_SIZE(lz->saved) == 0) + return NULL; + item = PyList_GET_ITEM(lz->saved, lz->index); + lz->index++; + if (lz->index >= Py_SIZE(lz->saved)) + lz->index = 0; + Py_INCREF(item); + return item; } static PyObject * cycle_reduce(cycleobject *lz) { - /* Create a new cycle with the iterator tuple, then set - * the saved state on it. - */ - return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), - lz->it, lz->saved, lz->firstpass); + /* Create a new cycle with the iterator tuple, then set the saved state */ + if (lz->it == NULL) { + PyObject *it = PyObject_GetIter(lz->saved); + if (it == NULL) + return NULL; + if (lz->index != 0) { + _Py_IDENTIFIER(__setstate__); + PyObject *res = _PyObject_CallMethodId(it, &PyId___setstate__, + "n", lz->index); + if (res == NULL) { + Py_DECREF(it); + return NULL; + } + Py_DECREF(res); + } + return Py_BuildValue("O(N)(Oi)", Py_TYPE(lz), it, lz->saved, 1); + } + return Py_BuildValue("O(O)(Oi)", Py_TYPE(lz), lz->it, lz->saved, + lz->firstpass); } static PyObject * @@ -972,10 +991,11 @@ cycle_setstate(cycleobject *lz, PyObject *state) int firstpass; if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; + Py_INCREF(saved); Py_CLEAR(lz->saved); lz->saved = saved; - Py_XINCREF(lz->saved); lz->firstpass = firstpass != 0; + lz->index = 0; Py_RETURN_NONE; } -- cgit v0.12 From bb562a9a5659d7c97790454e1b76819614dead26 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 17:08:48 -0700 Subject: Add error information to a failing test to help diagnose a buildbot failure. --- Lib/test/test_symbol.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index 4475bbc..c126be3 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -38,7 +38,10 @@ class TestSymbolGeneration(unittest.TestCase): self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) + self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), + 'symbol stat: %r\ntest_py stat: %r\n' % + (os.stat(SYMBOL_FILE), + os.stat(TEST_PY_FILE))) if __name__ == "__main__": -- cgit v0.12 From 47d159f5220918f9c1216ba93583f3aa740a8944 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 16 Aug 2015 19:43:34 -0700 Subject: Issue #24379: Add operator.subscript() as a convenience for building slices. --- Doc/library/operator.rst | 15 +++++++++++++++ Doc/whatsnew/3.6.rst | 7 ++++++- Lib/operator.py | 28 +++++++++++++++++++++++++++- Lib/test/test_operator.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index c01e63b..0695391 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -333,6 +333,21 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] +.. data:: subscript + + A helper to turn subscript notation into indexing objects. This can be + used to create item access patterns ahead of time to pass them into + various subscriptable objects. + + For example: + + * ``subscript[5] == 5`` + * ``subscript[3:7:2] == slice(3, 7, 2)`` + * ``subscript[5, 8] == (5, 8)`` + + .. versionadded:: 3.6 + + .. function:: methodcaller(name[, args...]) Return a callable object that calls the method *name* on its operand. If diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index e2aa1e2..48ff38a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -95,7 +95,12 @@ New Modules Improved Modules ================ -* None yet. +operator +-------- + +* New object :data:`operator.subscript` makes it easier to create complex + indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` + (Contributed by Joe Jevnik in :issue:`24379`.) Optimizations diff --git a/Lib/operator.py b/Lib/operator.py index 0e2e53e..bc2a947 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,6 +408,32 @@ def ixor(a, b): return a +@object.__new__ # create a singleton instance +class subscript: + """ + A helper to turn subscript notation into indexing objects. This can be + used to create item access patterns ahead of time to pass them into + various subscriptable objects. + + For example: + subscript[5] == 5 + subscript[3:7:2] == slice(3, 7, 2) + subscript[5, 8] == (5, 8) + """ + __slots__ = () + + def __new__(cls): + raise TypeError("cannot create '{}' instances".format(cls.__name__)) + + @staticmethod + def __getitem__(key): + return key + + @staticmethod + def __reduce__(): + return 'subscript' + + try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 54fd1f4..27501c2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,5 +596,38 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator +class SubscriptTestCase: + def test_subscript(self): + subscript = self.module.subscript + self.assertIsNone(subscript[None]) + self.assertEqual(subscript[0], 0) + self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) + self.assertEqual( + subscript[0, ..., :2, ...], + (0, Ellipsis, slice(2), Ellipsis), + ) + + def test_pickle(self): + from operator import subscript + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertIs( + pickle.loads(pickle.dumps(subscript, proto)), + subscript, + ) + + def test_singleton(self): + with self.assertRaises(TypeError): + type(self.module.subscript)() + + def test_immutable(self): + with self.assertRaises(AttributeError): + self.module.subscript.attr = None + + +class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): + pass + + if __name__ == "__main__": unittest.main() -- cgit v0.12 From 5b798abf5b56887d8dcde4e9fa82e405ef4b1dd3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 17 Aug 2015 22:04:45 -0700 Subject: Issue #24878: Add docstrings to selected namedtuples --- Lib/aifc.py | 8 ++++++++ Lib/dis.py | 9 +++++++++ Lib/sched.py | 11 +++++++++++ Lib/selectors.py | 12 ++++++++++-- Lib/shutil.py | 3 +++ Lib/sndhdr.py | 12 ++++++++++++ 6 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 7ebdbeb..1556b53 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -257,6 +257,14 @@ from collections import namedtuple _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') +_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' +_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.framerate.__doc__ = 'Sampling frequency' +_aifc_params.nframes.__doc__ = 'Number of audio frames' +_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' +_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type +('not compressed' for AIFF files)'""") + class Aifc_read: # Variables used in this class: diff --git a/Lib/dis.py b/Lib/dis.py index af37cdf..3540b47 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -162,6 +162,15 @@ def show_code(co, *, file=None): _Instruction = collections.namedtuple("_Instruction", "opname opcode arg argval argrepr offset starts_line is_jump_target") +_Instruction.opname.__doc__ = "Human readable name for operation" +_Instruction.opcode.__doc__ = "Numeric code for operation" +_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" +_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" +_Instruction.argrepr.__doc__ = "Human readable description of operation argument" +_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" +_Instruction.starts_line.__doc__ = "Line started by this opcode (if any), otherwise None" +_Instruction.is_jump_target.__doc__ = "True if other code jumps to here, otherwise False" + class Instruction(_Instruction): """Details for a bytecode operation diff --git a/Lib/sched.py b/Lib/sched.py index b47648d..bd7c0f1 100644 --- a/Lib/sched.py +++ b/Lib/sched.py @@ -46,6 +46,17 @@ class Event(namedtuple('Event', 'time, priority, action, argument, kwargs')): def __gt__(s, o): return (s.time, s.priority) > (o.time, o.priority) def __ge__(s, o): return (s.time, s.priority) >= (o.time, o.priority) +Event.time.__doc__ = ('''Numeric type compatible with the return value of the +timefunc function passed to the constructor.''') +Event.priority.__doc__ = ('''Events scheduled for the same time will be executed +in the order of their priority.''') +Event.action.__doc__ = ('''Executing the event means executing +action(*argument, **kwargs)''') +Event.argument.__doc__ = ('''argument is a sequence holding the positional +arguments for the action.''') +Event.kwargs.__doc__ = ('''kwargs is a dictionary holding the keyword +arguments for the action.''') + _sentinel = object() class scheduler: diff --git a/Lib/selectors.py b/Lib/selectors.py index 6d569c3..ecd8632 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -43,9 +43,17 @@ def _fileobj_to_fd(fileobj): SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) -"""Object used to associate a file object to its backing file descriptor, -selected event mask and attached data.""" +SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) + + Object used to associate a file object to its backing + file descriptor, selected event mask, and attached data. +""" +SelectorKey.fileobj.__doc__ = 'File object registered.' +SelectorKey.fd.__doc__ = 'Underlying file descriptor.' +SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' +SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. +For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" diff --git a/Lib/shutil.py b/Lib/shutil.py index a5da587..8edabdb 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -965,6 +965,9 @@ if hasattr(os, 'statvfs'): __all__.append('disk_usage') _ntuple_diskusage = collections.namedtuple('usage', 'total used free') + _ntuple_diskusage.total.__doc__ = 'Total space in bytes' + _ntuple_diskusage.used.__doc__ = 'Used space in bytes' + _ntuple_diskusage.free.__doc__ = 'Free space in bytes' def disk_usage(path): """Return disk usage statistics about the given path. diff --git a/Lib/sndhdr.py b/Lib/sndhdr.py index e5901ec..7ecafb4 100644 --- a/Lib/sndhdr.py +++ b/Lib/sndhdr.py @@ -37,6 +37,18 @@ from collections import namedtuple SndHeaders = namedtuple('SndHeaders', 'filetype framerate nchannels nframes sampwidth') +SndHeaders.filetype.__doc__ = ("""The value for type indicates the data type +and will be one of the strings 'aifc', 'aiff', 'au','hcom', +'sndr', 'sndt', 'voc', 'wav', '8svx', 'sb', 'ub', or 'ul'.""") +SndHeaders.framerate.__doc__ = ("""The sampling_rate will be either the actual +value or 0 if unknown or difficult to decode.""") +SndHeaders.nchannels.__doc__ = ("""The number of channels or 0 if it cannot be +determined or if the value is difficult to decode.""") +SndHeaders.nframes.__doc__ = ("""The value for frames will be either the number +of frames or -1.""") +SndHeaders.sampwidth.__doc__ = ("""Either the sample size in bits or +'A' for A-LAW or 'U' for u-LAW.""") + def what(filename): """Guess the type of a sound file.""" res = whathdr(filename) -- cgit v0.12 From a6ea44aed146d571a7236f2089255625c3c83693 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 17 Aug 2015 23:55:28 -0700 Subject: Minor cleanups --- Modules/itertoolsmodule.c | 156 ++++++++++++++++++++++------------------------ 1 file changed, 74 insertions(+), 82 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 61ad0c8..c7e1919 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1,17 +1,16 @@ #define PY_SSIZE_T_CLEAN - #include "Python.h" #include "structmember.h" /* Itertools module written and maintained by Raymond D. Hettinger - Copyright (c) 2003-2013 Python Software Foundation. + Copyright (c) 2003-2015 Python Software Foundation. All rights reserved. */ -/* groupby object ***********************************************************/ +/* groupby object ************************************************************/ typedef struct { PyObject_HEAD @@ -89,8 +88,7 @@ groupby_next(groupbyobject *gbo) else { int rcmp; - rcmp = PyObject_RichCompareBool(gbo->tgtkey, - gbo->currkey, Py_EQ); + rcmp = PyObject_RichCompareBool(gbo->tgtkey, gbo->currkey, Py_EQ); if (rcmp == -1) return NULL; else if (rcmp == 0) @@ -105,8 +103,7 @@ groupby_next(groupbyobject *gbo) newkey = newvalue; Py_INCREF(newvalue); } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, - newvalue, NULL); + newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); if (newkey == NULL) { Py_DECREF(newvalue); return NULL; @@ -303,8 +300,7 @@ _grouper_next(_grouperobject *igo) newkey = newvalue; Py_INCREF(newvalue); } else { - newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, - newvalue, NULL); + newkey = PyObject_CallFunctionObjArgs(gbo->keyfunc, newvalue, NULL); if (newkey == NULL) { Py_DECREF(newvalue); return NULL; @@ -332,8 +328,7 @@ _grouper_next(_grouperobject *igo) static PyObject * _grouper_reduce(_grouperobject *lz) { - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->parent, lz->tgtkey); + return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->parent, lz->tgtkey); } static PyMethodDef _grouper_methods[] = { @@ -387,8 +382,7 @@ static PyTypeObject _grouper_type = { }; - -/* tee object and with supporting function and objects ***************/ +/* tee object and with supporting function and objects ***********************/ /* The teedataobject pre-allocates space for LINKCELLS number of objects. To help the object fit neatly inside cache lines (space for 16 to 32 @@ -403,7 +397,7 @@ static PyTypeObject _grouper_type = { typedef struct { PyObject_HEAD PyObject *it; - int numread; /* 0 <= numread <= LINKCELLS */ + int numread; /* 0 <= numread <= LINKCELLS */ PyObject *nextlink; PyObject *(values[LINKCELLS]); } teedataobject; @@ -411,7 +405,7 @@ typedef struct { typedef struct { PyObject_HEAD teedataobject *dataobj; - int index; /* 0 <= index <= LINKCELLS */ + int index; /* 0 <= index <= LINKCELLS */ PyObject *weakreflist; } teeobject; @@ -468,6 +462,7 @@ static int teedataobject_traverse(teedataobject *tdo, visitproc visit, void * arg) { int i; + Py_VISIT(tdo->it); for (i = 0; i < tdo->numread; i++) Py_VISIT(tdo->values[i]); @@ -517,6 +512,7 @@ teedataobject_reduce(teedataobject *tdo) int i; /* create a temporary list of already iterated values */ PyObject *values = PyList_New(tdo->numread); + if (!values) return NULL; for (i=0 ; inumread ; i++) { @@ -859,7 +855,7 @@ PyDoc_STRVAR(tee_doc, "tee(iterable, n=2) --> tuple of n independent iterators."); -/* cycle object **********************************************************/ +/* cycle object **************************************************************/ typedef struct { PyObject_HEAD @@ -989,6 +985,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) { PyObject *saved=NULL; int firstpass; + if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) return NULL; Py_INCREF(saved); @@ -1064,7 +1061,7 @@ typedef struct { PyObject_HEAD PyObject *func; PyObject *it; - long start; + long start; } dropwhileobject; static PyTypeObject dropwhile_type; @@ -1154,8 +1151,7 @@ dropwhile_next(dropwhileobject *lz) static PyObject * dropwhile_reduce(dropwhileobject *lz) { - return Py_BuildValue("O(OO)l", Py_TYPE(lz), - lz->func, lz->it, lz->start); + return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->start); } static PyObject * @@ -1233,7 +1229,7 @@ typedef struct { PyObject_HEAD PyObject *func; PyObject *it; - long stop; + long stop; } takewhileobject; static PyTypeObject takewhile_type; @@ -1308,7 +1304,7 @@ takewhile_next(takewhileobject *lz) } ok = PyObject_IsTrue(good); Py_DECREF(good); - if (ok == 1) + if (ok > 0) return item; Py_DECREF(item); if (ok == 0) @@ -1319,14 +1315,14 @@ takewhile_next(takewhileobject *lz) static PyObject * takewhile_reduce(takewhileobject *lz) { - return Py_BuildValue("O(OO)l", Py_TYPE(lz), - lz->func, lz->it, lz->stop); + return Py_BuildValue("O(OO)l", Py_TYPE(lz), lz->func, lz->it, lz->stop); } static PyObject * takewhile_reduce_setstate(takewhileobject *lz, PyObject *state) { int stop = PyObject_IsTrue(state); + if (stop < 0) return NULL; lz->stop = stop; @@ -1391,7 +1387,7 @@ static PyTypeObject takewhile_type = { }; -/* islice object ************************************************************/ +/* islice object *************************************************************/ typedef struct { PyObject_HEAD @@ -1549,6 +1545,7 @@ islice_reduce(isliceobject *lz) * then 'setstate' with the next and count */ PyObject *stop; + if (lz->it == NULL) { PyObject *empty_list; PyObject *empty_it; @@ -1578,6 +1575,7 @@ static PyObject * islice_setstate(isliceobject *lz, PyObject *state) { Py_ssize_t cnt = PyLong_AsSsize_t(state); + if (cnt == -1 && PyErr_Occurred()) return NULL; lz->cnt = cnt; @@ -1792,7 +1790,7 @@ static PyTypeObject starmap_type = { }; -/* chain object ************************************************************/ +/* chain object **************************************************************/ typedef struct { PyObject_HEAD @@ -1919,6 +1917,7 @@ static PyObject * chain_setstate(chainobject *lz, PyObject *state) { PyObject *source, *active=NULL; + if (! PyArg_ParseTuple(state, "O|O", &source, &active)) return NULL; @@ -1945,8 +1944,8 @@ Alternate chain() contructor taking a single iterable argument\n\ that evaluates lazily."); static PyMethodDef chain_methods[] = { - {"from_iterable", (PyCFunction) chain_new_from_iterable, METH_O | METH_CLASS, - chain_from_iterable_doc}, + {"from_iterable", (PyCFunction) chain_new_from_iterable, METH_O | METH_CLASS, + chain_from_iterable_doc}, {"__reduce__", (PyCFunction)chain_reduce, METH_NOARGS, reduce_doc}, {"__setstate__", (PyCFunction)chain_setstate, METH_O, @@ -2003,10 +2002,10 @@ static PyTypeObject chain_type = { typedef struct { PyObject_HEAD - PyObject *pools; /* tuple of pool tuples */ - Py_ssize_t *indices; /* one index per pool */ - PyObject *result; /* most recently returned result tuple */ - int stopped; /* set to 1 when the product iterator is exhausted */ + PyObject *pools; /* tuple of pool tuples */ + Py_ssize_t *indices; /* one index per pool */ + PyObject *result; /* most recently returned result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } productobject; static PyTypeObject product_type; @@ -2025,7 +2024,8 @@ product_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *tmpargs = PyTuple_New(0); if (tmpargs == NULL) return NULL; - if (!PyArg_ParseTupleAndKeywords(tmpargs, kwds, "|n:product", kwlist, &repeat)) { + if (!PyArg_ParseTupleAndKeywords(tmpargs, kwds, "|n:product", + kwlist, &repeat)) { Py_DECREF(tmpargs); return NULL; } @@ -2349,15 +2349,15 @@ static PyTypeObject product_type = { }; -/* combinations object ************************************************************/ +/* combinations object *******************************************************/ typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the combinations iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } combinationsobject; static PyTypeObject combinations_type; @@ -2567,17 +2567,16 @@ combinations_setstate(combinationsobject *lz, PyObject *state) Py_ssize_t i; Py_ssize_t n = PyTuple_GET_SIZE(lz->pool); - if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) - { + if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != lz->r) { PyErr_SetString(PyExc_ValueError, "invalid arguments"); return NULL; } - for (i=0; ir; i++) - { + for (i=0; ir; i++) { Py_ssize_t max; PyObject* indexObject = PyTuple_GET_ITEM(state, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); + if (index == -1 && PyErr_Occurred()) return NULL; /* not an integer */ max = i + n - lz->r; @@ -2664,7 +2663,7 @@ static PyTypeObject combinations_type = { }; -/* combinations with replacement object *******************************************/ +/* combinations with replacement object **************************************/ /* Equivalent to: @@ -2694,11 +2693,11 @@ static PyTypeObject combinations_type = { */ typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per result element */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the cwr iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per result element */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the cwr iterator is exhausted */ } cwrobject; static PyTypeObject cwr_type; @@ -2715,8 +2714,9 @@ cwr_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_ssize_t i; static char *kwargs[] = {"iterable", "r", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "On:combinations_with_replacement", kwargs, - &iterable, &r)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "On:combinations_with_replacement", + kwargs, &iterable, &r)) return NULL; pool = PySequence_Tuple(iterable); @@ -2881,8 +2881,7 @@ cwr_reduce(cwrobject *lz) indices = PyTuple_New(lz->r); if (!indices) return NULL; - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* index = PyLong_FromSsize_t(lz->indices[i]); if (!index) { Py_DECREF(indices); @@ -2908,10 +2907,10 @@ cwr_setstate(cwrobject *lz, PyObject *state) } n = PyTuple_GET_SIZE(lz->pool); - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* indexObject = PyTuple_GET_ITEM(state, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); + if (index < 0 && PyErr_Occurred()) return NULL; /* not an integer */ /* clamp the index */ @@ -2996,7 +2995,7 @@ static PyTypeObject cwr_type = { }; -/* permutations object ************************************************************ +/* permutations object ******************************************************** def permutations(iterable, r=None): 'permutations(range(3), 2) --> (0,1) (0,2) (1,0) (1,2) (2,0) (2,1)' @@ -3023,12 +3022,12 @@ def permutations(iterable, r=None): typedef struct { PyObject_HEAD - PyObject *pool; /* input converted to a tuple */ - Py_ssize_t *indices; /* one index per element in the pool */ - Py_ssize_t *cycles; /* one rollover counter per element in the result */ - PyObject *result; /* most recently returned result tuple */ - Py_ssize_t r; /* size of result tuple */ - int stopped; /* set to 1 when the permutations iterator is exhausted */ + PyObject *pool; /* input converted to a tuple */ + Py_ssize_t *indices; /* one index per element in the pool */ + Py_ssize_t *cycles; /* one rollover counter per element in the result */ + PyObject *result; /* most recently returned result tuple */ + Py_ssize_t r; /* size of result tuple */ + int stopped; /* set to 1 when the iterator is exhausted */ } permutationsobject; static PyTypeObject permutations_type; @@ -3243,7 +3242,7 @@ permutations_reduce(permutationsobject *po) indices = PyTuple_New(n); if (indices == NULL) goto err; - for (i=0; iindices[i]); if (!index) goto err; @@ -3253,8 +3252,7 @@ permutations_reduce(permutationsobject *po) cycles = PyTuple_New(po->r); if (cycles == NULL) goto err; - for (i=0; ir; i++) - { + for (i=0 ; ir ; i++) { PyObject* index = PyLong_FromSsize_t(po->cycles[i]); if (!index) goto err; @@ -3282,15 +3280,12 @@ permutations_setstate(permutationsobject *po, PyObject *state) return NULL; n = PyTuple_GET_SIZE(po->pool); - if (PyTuple_GET_SIZE(indices) != n || - PyTuple_GET_SIZE(cycles) != po->r) - { + if (PyTuple_GET_SIZE(indices) != n || PyTuple_GET_SIZE(cycles) != po->r) { PyErr_SetString(PyExc_ValueError, "invalid arguments"); return NULL; } - for (i=0; iindices[i] = index; } - for (i=0; ir; i++) - { + for (i=0; ir; i++) { PyObject* indexObject = PyTuple_GET_ITEM(cycles, i); Py_ssize_t index = PyLong_AsSsize_t(indexObject); if (index < 0 && PyErr_Occurred()) @@ -3388,7 +3382,7 @@ static PyTypeObject permutations_type = { PyObject_GC_Del, /* tp_free */ }; -/* accumulate object ************************************************************/ +/* accumulate object ********************************************************/ typedef struct { PyObject_HEAD @@ -3489,7 +3483,7 @@ accumulate_reduce(accumulateobject *lz) return Py_BuildValue("O(OO)O", Py_TYPE(lz), lz->it, lz->binop?lz->binop:Py_None, lz->total?lz->total:Py_None); - } +} static PyObject * accumulate_setstate(accumulateobject *lz, PyObject *state) @@ -3652,7 +3646,7 @@ compress_next(compressobject *lz) ok = PyObject_IsTrue(selector); Py_DECREF(selector); - if (ok == 1) + if (ok > 0) return datum; Py_DECREF(datum); if (ok < 0) @@ -3665,7 +3659,7 @@ compress_reduce(compressobject *lz) { return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->data, lz->selectors); - } +} static PyMethodDef compress_methods[] = { {"__reduce__", (PyCFunction)compress_reduce, METH_NOARGS, @@ -3821,9 +3815,8 @@ filterfalse_next(filterfalseobject *lz) static PyObject * filterfalse_reduce(filterfalseobject *lz) { - return Py_BuildValue("O(OO)", Py_TYPE(lz), - lz->func, lz->it); - } + return Py_BuildValue("O(OO)", Py_TYPE(lz), lz->func, lz->it); +} static PyMethodDef filterfalse_methods[] = { {"__reduce__", (PyCFunction)filterfalse_reduce, METH_NOARGS, @@ -4274,9 +4267,7 @@ static PyTypeObject repeat_type = { PyObject_GC_Del, /* tp_free */ }; -/* ziplongest object ************************************************************/ - -#include "Python.h" +/* ziplongest object *********************************************************/ typedef struct { PyObject_HEAD @@ -4315,7 +4306,7 @@ zip_longest_new(PyTypeObject *type, PyObject *args, PyObject *kwds) ittuple = PyTuple_New(tuplesize); if (ittuple == NULL) return NULL; - for (i=0; i < tuplesize; ++i) { + for (i=0; i < tuplesize; i++) { PyObject *item = PyTuple_GET_ITEM(args, i); PyObject *it = PyObject_GetIter(item); if (it == NULL) { @@ -4456,6 +4447,7 @@ zip_longest_reduce(ziplongestobject *lz) */ int i; PyObject *args = PyTuple_New(PyTuple_GET_SIZE(lz->ittuple)); + if (args == NULL) return NULL; for (i=0; iittuple); i++) { -- cgit v0.12 From f1094140944721ca852d49fb30e107d4ffec6f12 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 00:20:20 -0700 Subject: Inline PyIter_Next() matching the other itertools code. --- Python/bltinmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2f22209..78ec3cb 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1164,7 +1164,8 @@ map_next(mapobject *lz) return NULL; for (i=0 ; iiters, i)); + PyObject *it = PyTuple_GET_ITEM(lz->iters, i); + val = Py_TYPE(it)->tp_iternext(it); if (val == NULL) { Py_DECREF(argtuple); return NULL; -- cgit v0.12 From 507343a2ef4f2992e2c318517b5833d1b1a1fddb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 00:35:52 -0700 Subject: Add missing docstring --- Lib/urllib/request.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a7fd017..e6abf34 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -138,6 +138,71 @@ __version__ = sys.version[:3] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *, cafile=None, capath=None, cadefault=False, context=None): + '''Open the URL url, which can be either a string or a Request object. + + *data* must be a bytes object specifying additional data to be sent to the + server, or None if no such data is needed. data may also be an iterable + object and in that case Content-Length value must be specified in the + headers. Currently HTTP requests are the only ones that use data; the HTTP + request will be a POST instead of a GET when the data parameter is + provided. + + *data* should be a buffer in the standard application/x-www-form-urlencoded + format. The urllib.parse.urlencode() function takes a mapping or sequence + of 2-tuples and returns a string in this format. It should be encoded to + bytes before being used as the data parameter. The charset parameter in + Content-Type header may be used to specify the encoding. If charset + parameter is not sent with the Content-Type header, the server following + the HTTP 1.1 recommendation may assume that the data is encoded in + ISO-8859-1 encoding. It is advisable to use charset parameter with encoding + used in Content-Type header with the Request. + + urllib.request module uses HTTP/1.1 and includes a "Connection:close" + header in its HTTP requests. + + The optional *timeout* parameter specifies a timeout in seconds for + blocking operations like the connection attempt (if not specified, the + global default timeout setting will be used). This only works for HTTP, + HTTPS and FTP connections. + + If *context* is specified, it must be a ssl.SSLContext instance describing + the various SSL options. See HTTPSConnection for more details. + + The optional *cafile* and *capath* parameters specify a set of trusted CA + certificates for HTTPS requests. cafile should point to a single file + containing a bundle of CA certificates, whereas capath should point to a + directory of hashed certificate files. More information can be found in + ssl.SSLContext.load_verify_locations(). + + The *cadefault* parameter is ignored. + + For http and https urls, this function returns a http.client.HTTPResponse + object which has the following HTTPResponse Objects methods. + + For ftp, file, and data urls and requests explicitly handled by legacy + URLopener and FancyURLopener classes, this function returns a + urllib.response.addinfourl object which can work as context manager and has + methods such as: + + * geturl() — return the URL of the resource retrieved, commonly used to + determine if a redirect was followed + + * info() — return the meta-information of the page, such as headers, in the + form of an email.message_from_string() instance (see Quick Reference to + HTTP Headers) + + * getcode() – return the HTTP status code of the response. Raises URLError + on errors. + + Note that *None& may be returned if no handler handles the request (though + the default installed global OpenerDirector uses UnknownHandler to ensure + this never happens). + + In addition, if proxy settings are detected (for example, when a *_proxy + environment variable like http_proxy is set), ProxyHandler is default + installed and makes sure the requests are handled through the proxy. + + ''' global _opener if cafile or capath or cadefault: if context is not None: -- cgit v0.12 From 501b4a73980d363fca3a69d5999d377f3ae5cc3f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 08:07:16 -0700 Subject: Use PyTuple_GET_SIZE like the adjacent code does. --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 78ec3cb..2038d2b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1158,7 +1158,7 @@ map_next(mapobject *lz) PyObject *result; Py_ssize_t numargs, i; - numargs = PyTuple_Size(lz->iters); + numargs = PyTuple_GET_SIZE(lz->iters); argtuple = PyTuple_New(numargs); if (argtuple == NULL) return NULL; -- cgit v0.12 From 631a35c71a0c1f4fb31aad04407825cee2fe8a48 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 19 Aug 2015 12:20:37 +1200 Subject: Issue #24054: decouple linecache tests from inspect tests Patch from David D. Riddle --- Lib/test/test_linecache.py | 74 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 21ef738..e74aef3 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -3,6 +3,7 @@ import linecache import unittest import os.path +import tempfile from test import support @@ -10,8 +11,6 @@ FILENAME = linecache.__file__ NONEXISTENT_FILENAME = FILENAME + '.missing' INVALID_NAME = '!@$)(!@#_1' EMPTY = '' -TESTS = 'inspect_fodder inspect_fodder2 mapping_tests' -TESTS = TESTS.split() TEST_PATH = os.path.dirname(__file__) MODULES = "linecache abc".split() MODULE_PATH = os.path.dirname(FILENAME) @@ -37,6 +36,65 @@ def f(): return 3''' # No ending newline +class TempFile: + + def setUp(self): + super().setUp() + with tempfile.NamedTemporaryFile(delete=False) as fp: + self.file_name = fp.name + fp.write(self.file_byte_string) + self.addCleanup(support.unlink, self.file_name) + + +class GetLineTestsGoodData(TempFile): + # file_list = ['list\n', 'of\n', 'good\n', 'strings\n'] + + def setUp(self): + self.file_byte_string = ''.join(self.file_list).encode('utf-8') + super().setUp() + + def test_getline(self): + with open(self.file_name) as fp: + for index, line in enumerate(fp): + if not line.endswith('\n'): + line += '\n' + + cached_line = linecache.getline(self.file_name, index + 1) + self.assertEqual(line, cached_line) + + def test_getlines(self): + lines = linecache.getlines(self.file_name) + self.assertEqual(lines, self.file_list) + + +class GetLineTestsBadData(TempFile): + # file_byte_string = b'Bad data goes here' + + def test_getline(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getline, self.file_name, 1) + + def test_getlines(self): + self.assertRaises((SyntaxError, UnicodeDecodeError), + linecache.getlines, self.file_name) + + +class EmptyFile(GetLineTestsGoodData, unittest.TestCase): + file_list = [] + + +class SingleEmptyLine(GetLineTestsGoodData, unittest.TestCase): + file_list = ['\n'] + + +class GoodUnicode(GetLineTestsGoodData, unittest.TestCase): + file_list = ['á\n', 'b\n', 'abcdef\n', 'ááááá\n'] + + +class BadUnicode(GetLineTestsBadData, unittest.TestCase): + file_byte_string = b'\x80abc' + + class LineCacheTests(unittest.TestCase): def test_getline(self): @@ -53,13 +111,6 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(getline(EMPTY, 1), EMPTY) self.assertEqual(getline(INVALID_NAME, 1), EMPTY) - # Check whether lines correspond to those from file iteration - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' - with open(filename) as file: - for index, line in enumerate(file): - self.assertEqual(line, getline(filename, index + 1)) - # Check module loading for entry in MODULES: filename = os.path.join(MODULE_PATH, entry) + '.py' @@ -80,12 +131,13 @@ class LineCacheTests(unittest.TestCase): def test_clearcache(self): cached = [] - for entry in TESTS: - filename = os.path.join(TEST_PATH, entry) + '.py' + for entry in MODULES: + filename = os.path.join(MODULE_PATH, entry) + '.py' cached.append(filename) linecache.getline(filename, 1) # Are all files cached? + self.assertNotEqual(cached, []) cached_empty = [fn for fn in cached if fn not in linecache.cache] self.assertEqual(cached_empty, []) -- cgit v0.12 From 15b87bfedcd40f601cb3c9caf75d2366c48e19fc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:03:08 -0700 Subject: Add in missing docstrings. --- Lib/http/client.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Lib/http/client.py b/Lib/http/client.py index 80c80cf..155c2e3 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -405,6 +405,7 @@ class HTTPResponse(io.BufferedIOBase): self.fp.flush() def readable(self): + """Always returns True""" return True # End of "raw stream" methods @@ -452,6 +453,10 @@ class HTTPResponse(io.BufferedIOBase): return s def readinto(self, b): + """Read up to len(b) bytes into bytearray b and return the number + of bytes read. + """ + if self.fp is None: return 0 @@ -683,6 +688,17 @@ class HTTPResponse(io.BufferedIOBase): return self.fp.fileno() def getheader(self, name, default=None): + '''Returns the value of the header matching *name*. + + If there are multiple matching headers, the values are + combined into a single string separated by commas and spaces. + + If no matching header is found, returns *default* or None if + the *default* is not specified. + + If the headers are unknown, raises http.client.ResponseNotReady. + + ''' if self.headers is None: raise ResponseNotReady() headers = self.headers.get_all(name) or default @@ -705,12 +721,45 @@ class HTTPResponse(io.BufferedIOBase): # For compatibility with old-style urllib responses. def info(self): + '''Returns an instance of the class mimetools.Message containing + meta-information associated with the URL. + + When the method is HTTP, these headers are those returned by + the server at the head of the retrieved HTML page (including + Content-Length and Content-Type). + + When the method is FTP, a Content-Length header will be + present if (as is now usual) the server passed back a file + length in response to the FTP retrieval request. A + Content-Type header will be present if the MIME type can be + guessed. + + When the method is local-file, returned headers will include + a Date representing the file’s last-modified time, a + Content-Length giving file size, and a Content-Type + containing a guess at the file’s type. See also the + description of the mimetools module. + + ''' return self.headers def geturl(self): + '''Return the real URL of the page. + + In some cases, the HTTP server redirects a client to another + URL. The urlopen() function handles this transparently, but in + some cases the caller needs to know which URL the client was + redirected to. The geturl() method can be used to get at this + redirected URL. + + ''' return self.url def getcode(self): + '''Return the HTTP status code that was sent with the response, + or None if the URL is not an HTTP URL. + + ''' return self.status class HTTPConnection: -- cgit v0.12 From 95801bbe4e96aeb706c8fcf9fb3698c39a72b847 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 18 Aug 2015 22:25:16 -0700 Subject: Issue #24879: Teach pydoc to display named tuple fields in the order they were defined. --- Lib/pydoc.py | 19 +++++++++++++++---- Lib/test/test_pydoc.py | 16 ++++++++++++++++ Misc/NEWS | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index ee558bf..f4f2530 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -209,6 +209,18 @@ def classify_class_attrs(object): results.append((name, kind, cls, value)) return results +def sort_attributes(attrs, object): + 'Sort the attrs list in-place by _fields and then alphabetically by name' + # This allows data descriptors to be ordered according + # to a _fields attribute if present. + fields = getattr(object, '_fields', []) + try: + field_order = {name : i-len(fields) for (i, name) in enumerate(fields)} + except TypeError: + field_order = {} + keyfunc = lambda attr: (field_order.get(attr[0], 0), attr[0]) + attrs.sort(key=keyfunc) + # ----------------------------------------------------- module manipulation def ispackage(path): @@ -867,8 +879,7 @@ class HTMLDoc(Doc): object.__module__) tag += ':
\n' - # Sort attrs by name. - attrs.sort(key=lambda t: t[0]) + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill('Methods %s' % tag, attrs, @@ -1286,8 +1297,8 @@ location listed above. else: tag = "inherited from %s" % classname(thisclass, object.__module__) - # Sort attrs by name. - attrs.sort() + + sort_attributes(attrs, object) # Pump out the attrs, segregated by kind. attrs = spill("Methods %s:\n" % tag, attrs, diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index ec5c31b..0533a03 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -811,6 +811,22 @@ class TestDescriptions(unittest.TestCase): self.assertEqual(self._get_summary_line(t.wrap), "wrap(text) method of textwrap.TextWrapper instance") + def test_field_order_for_named_tuples(self): + Person = namedtuple('Person', ['nickname', 'firstname', 'agegroup']) + s = pydoc.render_doc(Person) + self.assertLess(s.index('nickname'), s.index('firstname')) + self.assertLess(s.index('firstname'), s.index('agegroup')) + + class NonIterableFields: + _fields = None + + class NonHashableFields: + _fields = [[]] + + # Make sure these doesn't fail + pydoc.render_doc(NonIterableFields) + pydoc.render_doc(NonHashableFields) + @requires_docstrings def test_bound_builtin_method(self): s = StringIO() diff --git a/Misc/NEWS b/Misc/NEWS index b2ec4f9..0bb5697 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,6 +26,10 @@ Library header in part headers. Patch written by Peter Landry and reviewed by Pierre Quentel. +- Issue #24879: help() and pydoc can now list named tuple fields in the + order they were defined rather than alphabetically. The ordering is + determined by the _fields attribute if present. + - Issue #24774: Fix docstring in http.server.test. Patch from Chiu-Hsiang Hsu. - Issue #21159: Improve message in configparser.InterpolationMissingOptionError. -- cgit v0.12 From e4d35dc8b6fb3c071d85e2d8830566876288c180 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Thu, 20 Aug 2015 10:48:46 +1200 Subject: Fix test_linecache change. In non-UTF8 locales open() will have the wrong encoding for the test data. We need to open the file via tokenize, to get a detected encoding instead. --- Lib/test/test_linecache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e74aef3..240db7f 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,6 +4,7 @@ import linecache import unittest import os.path import tempfile +import tokenize from test import support @@ -54,7 +55,7 @@ class GetLineTestsGoodData(TempFile): super().setUp() def test_getline(self): - with open(self.file_name) as fp: + with tokenize.open(self.file_name) as fp: for index, line in enumerate(fp): if not line.endswith('\n'): line += '\n' -- cgit v0.12 From 163b5c668d33689aad9f2490bfc5318fba64fa47 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 21 Aug 2015 09:40:38 -0400 Subject: Fix a trivial typo. --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index c1c3907..8b356fe 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -511,7 +511,7 @@ PyAST_Validate(mod_ty mod) /* Data structure used internally */ struct compiling { char *c_encoding; /* source encoding */ - PyArena *c_arena; /* arena for allocating memeory */ + PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ PyObject *c_normalize_args; /* Normalization argument tuple. */ -- cgit v0.12 From 4e70772b52b21c39488261d66b3feae2b75352dc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 23 Aug 2015 11:28:01 -0700 Subject: Issue #24878: Fix typos and line wrap --- Lib/aifc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/aifc.py b/Lib/aifc.py index 1556b53..692d0bf 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -258,12 +258,13 @@ _aifc_params = namedtuple('_aifc_params', 'nchannels sampwidth framerate nframes comptype compname') _aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)' -_aifc_params.sampwidth.__doc__ = 'Ample width in bytes' +_aifc_params.sampwidth.__doc__ = 'Sample width in bytes' _aifc_params.framerate.__doc__ = 'Sampling frequency' _aifc_params.nframes.__doc__ = 'Number of audio frames' _aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)' -_aifc_params.compname.__doc__ = ("""A human-readable version ofcompression type -('not compressed' for AIFF files)'""") +_aifc_params.compname.__doc__ = ("""\ +A human-readable version of the compression type +('not compressed' for AIFF files)""") class Aifc_read: -- cgit v0.12 From 4bffa5fa3a4b70a149045a91d7dae8deb5a388ad Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 24 Aug 2015 11:46:56 +1200 Subject: Issue #24633: site-packages/README -> README.txt. --- Lib/site-packages/README | 2 -- Lib/site-packages/README.txt | 2 ++ Misc/NEWS | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 Lib/site-packages/README create mode 100644 Lib/site-packages/README.txt diff --git a/Lib/site-packages/README b/Lib/site-packages/README deleted file mode 100644 index 273f625..0000000 --- a/Lib/site-packages/README +++ /dev/null @@ -1,2 +0,0 @@ -This directory exists so that 3rd party packages can be installed -here. Read the source for site.py for more details. diff --git a/Lib/site-packages/README.txt b/Lib/site-packages/README.txt new file mode 100644 index 0000000..273f625 --- /dev/null +++ b/Lib/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/Misc/NEWS b/Misc/NEWS index 7b11b88..5207e83 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -22,6 +22,8 @@ Core and Builtins Library ------- +- Issue #24633: site-packages/README -> README.txt. + - Issue #20362: Honour TestCase.longMessage correctly in assertRegex. Patch from Ilia Kurenkov. -- cgit v0.12 From 69de2a576cde2d1cfeac67472ad9d22aa1e87a4b Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Wed, 26 Aug 2015 12:40:28 +1200 Subject: Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. --- Lib/timeit.py | 32 +++++++++++++++++++------------- Misc/NEWS | 3 +++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/Lib/timeit.py b/Lib/timeit.py index 2de88f7..98cb3eb 100755 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -317,20 +317,26 @@ def main(args=None, *, _wrap_timer=None): print("%d loops," % number, end=' ') usec = best * 1e6 / number if time_unit is not None: - print("best of %d: %.*g %s per loop" % (repeat, precision, - usec/units[time_unit], time_unit)) + scale = units[time_unit] else: - if usec < 1000: - print("best of %d: %.*g usec per loop" % (repeat, precision, usec)) - else: - msec = usec / 1000 - if msec < 1000: - print("best of %d: %.*g msec per loop" % (repeat, - precision, msec)) - else: - sec = msec / 1000 - print("best of %d: %.*g sec per loop" % (repeat, - precision, sec)) + scales = [(scale, unit) for unit, scale in units.items()] + scales.sort(reverse=True) + for scale, time_unit in scales: + if usec >= scale: + break + print("best of %d: %.*g %s per loop" % (repeat, precision, + usec/scale, time_unit)) + best = min(r) + usec = best * 1e6 / number + worst = max(r) + if worst >= best * 4: + usec = worst * 1e6 / number + import warnings + warnings.warn_explicit( + "The test results are likely unreliable. The worst\n" + "time (%.*g %s) was more than four times slower than the best time." % + (precision, usec/scale, time_unit), + UserWarning, '', 0) return None if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index dabbeea..3404b09 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,9 @@ Core and Builtins Library ------- +- Issue #23552: Timeit now warns when there is substantial (4x) variance + between best and worst times. Patch from Serhiy Storchaka. + - Issue #24633: site-packages/README -> README.txt. - Issue #24879: help() and pydoc can now list named tuple fields in the -- cgit v0.12 From ab2aa6dc9197a3fc8fca5958a8578e697f88034e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 26 Aug 2015 14:10:32 -0400 Subject: Fixed an incorrect comment. --- Objects/stringlib/unicode_format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/stringlib/unicode_format.h b/Objects/stringlib/unicode_format.h index aec221a..d72e47d 100644 --- a/Objects/stringlib/unicode_format.h +++ b/Objects/stringlib/unicode_format.h @@ -67,7 +67,7 @@ SubString_new_object(SubString *str) return PyUnicode_Substring(str->str, str->start, str->end); } -/* return a new string. if str->str is NULL, return None */ +/* return a new string. if str->str is NULL, return a new empty string */ Py_LOCAL_INLINE(PyObject *) SubString_new_object_or_empty(SubString *str) { -- cgit v0.12 From 357cb986b2e330d63dbe3115b32a4acaa93e3918 Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Fri, 28 Aug 2015 16:56:45 -0400 Subject: Fixed typos in TZ format description --- Doc/library/time.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 3d335c8..73436ca 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -634,11 +634,11 @@ The module defines the following functions and data items: it is possible to refer to February 29. :samp:`M{m}.{n}.{d}` - The *d*'th day (0 <= *d* <= 6) or week *n* of month *m* of the year (1 + The *d*'th day (0 <= *d* <= 6) of week *n* of month *m* of the year (1 <= *n* <= 5, 1 <= *m* <= 12, where week 5 means "the last *d* day in month *m*" which may occur in either the fourth or the fifth week). Week 1 is the first week in which the *d*'th day occurs. Day - zero is Sunday. + zero is a Sunday. ``time`` has the same format as ``offset`` except that no leading sign ('-' or '+') is allowed. The default, if time is not given, is 02:00:00. -- cgit v0.12 From 53e137c8dde65eb0d48d26d262f1f6bc5d103e68 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 00:49:16 +0200 Subject: Refactor pytime.c Move code to convert double timestamp to subfunctions. --- Python/pytime.c | 113 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 02a9374..17ef197 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -61,43 +61,51 @@ _PyLong_FromTime_t(time_t t) } static int -_PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, +_PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { - assert(denominator <= LONG_MAX); - if (PyFloat_Check(obj)) { - double d, intpart, err; - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ - volatile double floatpart; + double intpart, err; + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double floatpart; + + floatpart = modf(d, &intpart); + if (floatpart < 0) { + floatpart = 1.0 + floatpart; + intpart -= 1.0; + } - d = PyFloat_AsDouble(obj); - floatpart = modf(d, &intpart); - if (floatpart < 0) { - floatpart = 1.0 + floatpart; - intpart -= 1.0; + floatpart *= denominator; + if (round == _PyTime_ROUND_CEILING) { + floatpart = ceil(floatpart); + if (floatpart >= denominator) { + floatpart = 0.0; + intpart += 1.0; } + } + else { + floatpart = floor(floatpart); + } - floatpart *= denominator; - if (round == _PyTime_ROUND_CEILING) { - floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { - floatpart = floor(floatpart); - } + *sec = (time_t)intpart; + err = intpart - (double)*sec; + if (err <= -1.0 || err >= 1.0) { + error_time_t_overflow(); + return -1; + } - *sec = (time_t)intpart; - err = intpart - (double)*sec; - if (err <= -1.0 || err >= 1.0) { - error_time_t_overflow(); - return -1; - } + *numerator = (long)floatpart; + return 0; +} - *numerator = (long)floatpart; - return 0; +static int +_PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, + double denominator, _PyTime_round_t round) +{ + assert(denominator <= LONG_MAX); + if (PyFloat_Check(obj)) { + double d = PyFloat_AsDouble(obj); + return _PyTime_DoubleToDenominator(d, sec, numerator, + denominator, round); } else { *sec = _PyLong_AsTime_t(obj); @@ -221,29 +229,38 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) #endif static int +_PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, + long to_nanoseconds) +{ + /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + volatile double d, err; + + /* convert to a number of nanoseconds */ + d = value; + d *= to_nanoseconds; + + if (round == _PyTime_ROUND_CEILING) + d = ceil(d); + else + d = floor(d); + + *t = (_PyTime_t)d; + err = d - (double)*t; + if (fabs(err) >= 1.0) { + _PyTime_overflow(); + return -1; + } + return 0; +} + +static int _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, long to_nanoseconds) { if (PyFloat_Check(obj)) { - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ - volatile double d, err; - - /* convert to a number of nanoseconds */ + double d; d = PyFloat_AsDouble(obj); - d *= to_nanoseconds; - - if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); - - *t = (_PyTime_t)d; - err = d - (double)*t; - if (fabs(err) >= 1.0) { - _PyTime_overflow(); - return -1; - } - return 0; + return _PyTime_FromFloatObject(t, d, round, to_nanoseconds); } else { #ifdef HAVE_LONG_LONG -- cgit v0.12 From bbdda21a7a54c30211b33ad736d7bbbf19ea08df Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 00:50:43 +0200 Subject: Move assertion inside _PyTime_ObjectToTimeval() Change also _PyTime_FromSeconds() assertion to ensure that the _PyTime_t type is used. --- Modules/_datetimemodule.c | 1 - Python/pytime.c | 20 ++++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 7e4be5b..5cff3f8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4100,7 +4100,6 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, if (_PyTime_ObjectToTimeval(timestamp, &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; - assert(0 <= us && us <= 999999); return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); } diff --git a/Python/pytime.c b/Python/pytime.c index 17ef197..fcac68a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -101,7 +101,8 @@ static int _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { - assert(denominator <= LONG_MAX); + assert(denominator <= (double)LONG_MAX); + if (PyFloat_Check(obj)) { double d = PyFloat_AsDouble(obj); return _PyTime_DoubleToDenominator(d, sec, numerator, @@ -149,14 +150,20 @@ int _PyTime_ObjectToTimespec(PyObject *obj, time_t *sec, long *nsec, _PyTime_round_t round) { - return _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); + int res; + res = _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); + assert(0 <= *nsec && *nsec < SEC_TO_NS ); + return res; } int _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, _PyTime_round_t round) { - return _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); + int res; + res = _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); + assert(0 <= *usec && *usec < SEC_TO_US ); + return res; } static void @@ -170,12 +177,13 @@ _PyTime_t _PyTime_FromSeconds(int seconds) { _PyTime_t t; + t = (_PyTime_t)seconds; /* ensure that integer overflow cannot happen, int type should have 32 bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_MS takes 30 bits). */ - assert((seconds >= 0 && seconds <= _PyTime_MAX / SEC_TO_NS) - || (seconds < 0 && seconds >= _PyTime_MIN / SEC_TO_NS)); - t = (_PyTime_t)seconds * SEC_TO_NS; + assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) + || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); + t *= SEC_TO_NS; return t; } -- cgit v0.12 From 744742320f259776dab1c8ef86e22225ef569d17 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 01:43:56 +0200 Subject: Issue #23517: Add "half up" rounding mode to the _PyTime API --- Include/pytime.h | 5 +++- Lib/test/test_time.py | 64 ++++++++++++++++++++++++++++++++++++++++++++--- Modules/_testcapimodule.c | 4 ++- Python/pytime.c | 64 +++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 122 insertions(+), 15 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 027c3d8..98ae12b 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -30,7 +30,10 @@ typedef enum { _PyTime_ROUND_FLOOR=0, /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ - _PyTime_ROUND_CEILING + _PyTime_ROUND_CEILING=1, + /* Round to nearest with ties going away from zero. + For example, used to round from a Python float. */ + _PyTime_ROUND_HALF_UP } _PyTime_round_t; /* Convert a time_t to a PyLong. */ diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6334e02..ed20470 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,8 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 + # Round to nearest with ties going away from zero + ROUND_HALF_UP = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING) +ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, + _PyTime.ROUND_HALF_UP) class TimeTestCase(unittest.TestCase): @@ -753,11 +756,11 @@ class TestPyTime_t(unittest.TestCase): (123.0, 123 * SEC_TO_NS), (-7.0, -7 * SEC_TO_NS), - # nanosecond are kept for value <= 2^23 seconds + # nanosecond are kept for value <= 2^23 seconds, + # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -790,24 +793,36 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), + ( 1e-10, 0, HALF_UP), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), + (-1e-10, 0, HALF_UP), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), + ( 1.1234567899, 1123456790, HALF_UP), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), + (-1.1234567899, -1123456790, HALF_UP), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), + ( 0.9999999999, 1000000000, HALF_UP), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), + (-0.9999999999, -1000000000, HALF_UP), + + # close to 2^23 seconds + (2**23 - 1e-9, 8388607999999999, FLOOR), + (2**23 - 1e-9, 8388607999999999, CEILING), + (2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -875,18 +890,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), + (1, (0, 0), HALF_UP), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), + (-1, (0, 0), HALF_UP), # seconds + nanoseconds (1234567001, (1, 234567), FLOOR), (1234567001, (1, 234568), CEILING), + (1234567001, (1, 234567), HALF_UP), (-1234567001, (-2, 765432), FLOOR), (-1234567001, (-2, 765433), CEILING), + (-1234567001, (-2, 765433), HALF_UP), + + # half up + (499, (0, 0), HALF_UP), + (500, (0, 1), HALF_UP), + (501, (0, 1), HALF_UP), + (999, (0, 1), HALF_UP), + (-499, (0, 0), HALF_UP), + (-500, (0, 0), HALF_UP), + (-501, (-1, 999999), HALF_UP), + (-999, (-1, 999999), HALF_UP), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -929,18 +959,33 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), + (1234 * MS_TO_NS + 1, 1234, HALF_UP), (-1234 * MS_TO_NS - 1, -1234, FLOOR), (-1234 * MS_TO_NS - 1, -1235, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + + # half up + (499999, 0, HALF_UP), + (499999, 0, HALF_UP), + (500000, 1, HALF_UP), + (999999, 1, HALF_UP), + (-499999, 0, HALF_UP), + (-500000, -1, HALF_UP), + (-500001, -1, HALF_UP), + (-999999, -1, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -966,18 +1011,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), + (1, 0, HALF_UP), (-1, 0, FLOOR), (-1, -1, CEILING), + (-1, 0, HALF_UP), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), + (1234 * US_TO_NS + 1, 1234, HALF_UP), (-1234 * US_TO_NS - 1, -1234, FLOOR), (-1234 * US_TO_NS - 1, -1235, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_UP), + + # half up + (1499, 1, HALF_UP), + (1500, 2, HALF_UP), + (1501, 2, HALF_UP), + (-1499, -1, HALF_UP), + (-1500, -2, HALF_UP), + (-1501, -2, HALF_UP), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ba0a24b..c38d9ae 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2646,7 +2646,9 @@ run_in_subinterp(PyObject *self, PyObject *args) static int check_time_rounding(int round) { - if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) { + if (round != _PyTime_ROUND_FLOOR + && round != _PyTime_ROUND_CEILING + && round != _PyTime_ROUND_HALF_UP) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index fcac68a..3294e0f 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,6 +60,17 @@ _PyLong_FromTime_t(time_t t) #endif } +static double +_PyTime_RoundHalfUp(double x) +{ + if (x >= 0.0) + x = floor(x + 0.5); + else + x = ceil(x - 0.5); + return x; +} + + static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -75,7 +86,9 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, } floatpart *= denominator; - if (round == _PyTime_ROUND_CEILING) { + if (round == _PyTime_ROUND_HALF_UP) + floatpart = _PyTime_RoundHalfUp(floatpart); + else if (round == _PyTime_ROUND_CEILING) { floatpart = ceil(floatpart); if (floatpart >= denominator) { floatpart = 0.0; @@ -124,7 +137,9 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) double d, intpart, err; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_CEILING) + if (round == _PyTime_ROUND_HALF_UP) + d = _PyTime_RoundHalfUp(d); + else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -247,7 +262,9 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, d = value; d *= to_nanoseconds; - if (round == _PyTime_ROUND_CEILING) + if (round == _PyTime_ROUND_HALF_UP) + d = _PyTime_RoundHalfUp(d); + else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else d = floor(d); @@ -333,7 +350,19 @@ static _PyTime_t _PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) { assert(k > 1); - if (round == _PyTime_ROUND_CEILING) { + if (round == _PyTime_ROUND_HALF_UP) { + _PyTime_t x, r; + x = t / k; + r = t % k; + if (Py_ABS(r) >= k / 2) { + if (t >= 0) + x++; + else + x--; + } + return x; + } + else if (round == _PyTime_ROUND_CEILING) { if (t >= 0) return (t + k - 1) / k; else @@ -359,8 +388,10 @@ static int _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { + const long k = US_TO_NS; _PyTime_t secs, ns; int res = 0; + int usec; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; @@ -392,20 +423,33 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (round == _PyTime_ROUND_CEILING) - tv->tv_usec = (int)((ns + US_TO_NS - 1) / US_TO_NS); + if (round == _PyTime_ROUND_HALF_UP) { + _PyTime_t r; + usec = (int)(ns / k); + r = ns % k; + if (Py_ABS(r) >= k / 2) { + if (ns >= 0) + usec++; + else + usec--; + } + } + else if (round == _PyTime_ROUND_CEILING) + usec = (int)((ns + k - 1) / k); else - tv->tv_usec = (int)(ns / US_TO_NS); + usec = (int)(ns / k); - if (tv->tv_usec >= SEC_TO_US) { - tv->tv_usec -= SEC_TO_US; + if (usec >= SEC_TO_US) { + usec -= SEC_TO_US; tv->tv_sec += 1; } if (res && raise) _PyTime_overflow(); - assert(0 <= tv->tv_usec && tv->tv_usec <= 999999); + assert(0 <= usec && usec <= 999999); + + tv->tv_usec = usec; return res; } -- cgit v0.12 From 265e1259e4ad7001ea366d52c56adde8daed7a69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 01:57:23 +0200 Subject: Issue #23517: datetime.datetime.fromtimestamp() and datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going away from zero, instead of rounding towards minus infinity (-inf), as Python 2 and Python older than 3.3. --- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 38f31e4..8226f20 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: datetime.datetime.fromtimestamp() and + datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going + away from zero, instead of rounding towards minus infinity (-inf), as Python + 2 and Python older than 3.3. + - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5cff3f8..e1cd2b5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4098,7 +4098,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_FLOOR) == -1) + &timet, &us, _PyTime_ROUND_HALF_UP) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From a53ec7a91abb7cdbb1a13862f39d3183cab7f2a4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:10:26 +0200 Subject: Backed out changeset b690bf218702 Issue #23517: the change broke test_datetime. datetime.timedelta() rounding mode must also be changed, and test_datetime must be updated for the new rounding mode (half up). --- Misc/NEWS | 5 ----- Modules/_datetimemodule.c | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 8226f20..38f31e4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,11 +17,6 @@ Core and Builtins Library ------- -- Issue #23517: datetime.datetime.fromtimestamp() and - datetime.datetime.utcfromtimestamp() now rounds to nearest with ties going - away from zero, instead of rounding towards minus infinity (-inf), as Python - 2 and Python older than 3.3. - - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e1cd2b5..5cff3f8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4098,7 +4098,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_HALF_UP) == -1) + &timet, &us, _PyTime_ROUND_FLOOR) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From 67edcc905d3574d87988537e1a5290a1c7717da0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:37:46 +0200 Subject: Issue #23517: Fix _PyTime_ObjectToDenominator() * initialize numerator on overflow error ensure that numerator is smaller than * denominator. --- Python/pytime.c | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 3294e0f..8a2579b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,6 +60,7 @@ _PyLong_FromTime_t(time_t t) #endif } +/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ static double _PyTime_RoundHalfUp(double x) { @@ -81,32 +82,31 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); if (floatpart < 0) { - floatpart = 1.0 + floatpart; + floatpart += 1.0; intpart -= 1.0; } floatpart *= denominator; if (round == _PyTime_ROUND_HALF_UP) floatpart = _PyTime_RoundHalfUp(floatpart); - else if (round == _PyTime_ROUND_CEILING) { + else if (round == _PyTime_ROUND_CEILING) floatpart = ceil(floatpart); - if (floatpart >= denominator) { - floatpart = 0.0; - intpart += 1.0; - } - } - else { + else floatpart = floor(floatpart); + if (floatpart >= denominator) { + floatpart -= denominator; + intpart += 1.0; } + assert(0.0 <= floatpart && floatpart < denominator); *sec = (time_t)intpart; + *numerator = (long)floatpart; + err = intpart - (double)*sec; if (err <= -1.0 || err >= 1.0) { error_time_t_overflow(); return -1; } - - *numerator = (long)floatpart; return 0; } @@ -123,9 +123,9 @@ _PyTime_ObjectToDenominator(PyObject *obj, time_t *sec, long *numerator, } else { *sec = _PyLong_AsTime_t(obj); + *numerator = 0; if (*sec == (time_t)-1 && PyErr_Occurred()) return -1; - *numerator = 0; return 0; } } @@ -167,7 +167,7 @@ _PyTime_ObjectToTimespec(PyObject *obj, time_t *sec, long *nsec, { int res; res = _PyTime_ObjectToDenominator(obj, sec, nsec, 1e9, round); - assert(0 <= *nsec && *nsec < SEC_TO_NS ); + assert(0 <= *nsec && *nsec < SEC_TO_NS); return res; } @@ -177,7 +177,7 @@ _PyTime_ObjectToTimeval(PyObject *obj, time_t *sec, long *usec, { int res; res = _PyTime_ObjectToDenominator(obj, sec, usec, 1e6, round); - assert(0 <= *usec && *usec < SEC_TO_US ); + assert(0 <= *usec && *usec < SEC_TO_US); return res; } @@ -444,12 +444,11 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, tv->tv_sec += 1; } + assert(0 <= usec && usec < SEC_TO_US); + tv->tv_usec = usec; + if (res && raise) _PyTime_overflow(); - - assert(0 <= usec && usec <= 999999); - - tv->tv_usec = usec; return res; } @@ -484,7 +483,7 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) } ts->tv_nsec = nsec; - assert(0 <= ts->tv_nsec && ts->tv_nsec <= 999999999); + assert(0 <= ts->tv_nsec && ts->tv_nsec < SEC_TO_NS); return 0; } #endif -- cgit v0.12 From acea9f6208c74186df62f8c4cfd269919e33779f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 10:39:40 +0200 Subject: Issue #23517: Reintroduce unit tests for the old PyTime API since it's still used. --- Lib/test/test_time.py | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index ed20470..926e700 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -727,6 +727,9 @@ class TestPytime(unittest.TestCase): @unittest.skipUnless(_testcapi is not None, 'need the _testcapi module') class TestPyTime_t(unittest.TestCase): + """ + Test the _PyTime_t API. + """ def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): @@ -1041,5 +1044,156 @@ class TestPyTime_t(unittest.TestCase): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) +@unittest.skipUnless(_testcapi is not None, + 'need the _testcapi module') +class TestOldPyTime(unittest.TestCase): + """ + Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + """ + def setUp(self): + self.invalid_values = ( + -(2 ** 100), 2 ** 100, + -(2.0 ** 100.0), 2.0 ** 100.0, + ) + + @support.cpython_only + def test_time_t(self): + from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, time_t in ( + # int + (0, 0), + (-1, -1), + + # float + (1.0, 1), + (-1.0, -1), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, time_t, rnd in ( + (-1.9, -2, FLOOR), + (-1.9, -2, HALF_UP), + (-1.9, -1, CEILING), + (1.9, 1, FLOOR), + (1.9, 2, HALF_UP), + (1.9, 2, CEILING), + ): + self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + + # Test OverflowError + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_time_t, invalid, rnd) + + def test_timeval(self): + from _testcapi import pytime_object_to_timeval + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timeval in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1.2, (-2, 800000)), + (-1e-6, (-1, 999999)), + (1e-6, (0, 1)), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), + timeval) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timeval, rnd in ( + (-1e-7, (-1, 999999), FLOOR), + (-1e-7, (0, 0), CEILING), + (-1e-7, (0, 0), HALF_UP), + + (1e-7, (0, 0), FLOOR), + (1e-7, (0, 1), CEILING), + (1e-7, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + (0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), + + (0.9999999, (0, 999999), FLOOR), + (0.9999999, (1, 0), CEILING), + (0.9999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timeval=timeval): + self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) + + rnd = _PyTime.ROUND_FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timeval, invalid, rnd) + + @support.cpython_only + def test_timespec(self): + from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (-1.2, (-2, 800000000)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), + timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP + for obj, timespec, rnd in ( + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (0, 0), HALF_UP), + + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + (0.9999999999, (1, 0), HALF_UP), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Test OverflowError + rnd = FLOOR + for invalid in self.invalid_values: + self.assertRaises(OverflowError, + pytime_object_to_timespec, invalid, rnd) + + if __name__ == "__main__": unittest.main() -- cgit v0.12 From ead144c19b2226f82f5c0d9ae3cb26b59dcbf0c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 11:05:32 +0200 Subject: test_time: add more tests on HALF_UP rounding mode --- Lib/test/test_time.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 926e700..db96277 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1082,9 +1082,18 @@ class TestOldPyTime(unittest.TestCase): (-1.9, -2, FLOOR), (-1.9, -2, HALF_UP), (-1.9, -1, CEILING), + (1.9, 1, FLOOR), (1.9, 2, HALF_UP), (1.9, 2, CEILING), + + (-0.6, -1, HALF_UP), + (-0.5, -1, HALF_UP), + (-0.4, 0, HALF_UP), + + (0.4, 0, HALF_UP), + (0.5, 1, HALF_UP), + (0.6, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1127,13 +1136,19 @@ class TestOldPyTime(unittest.TestCase): (1e-7, (0, 1), CEILING), (1e-7, (0, 0), HALF_UP), - (0.4e-6, (0, 0), HALF_UP), - (0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), - (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), (0.9999999, (1, 0), HALF_UP), + + (-0.6e-6, (-1, 999999), HALF_UP), + # skipped, -0.5e-6 is inexact in base 2 + #(-0.5e-6, (-1, 999999), HALF_UP), + (-0.4e-6, (0, 0), HALF_UP), + + (0.4e-6, (0, 0), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(0.5e-6, (0, 1), HALF_UP), + (0.6e-6, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1177,13 +1192,18 @@ class TestOldPyTime(unittest.TestCase): (1e-10, (0, 1), CEILING), (1e-10, (0, 0), HALF_UP), - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), - (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), (0.9999999999, (1, 0), HALF_UP), + + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) -- cgit v0.12 From 24b822e21e19984eadd20e062a11ce7182cb81da Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 11:58:56 +0200 Subject: Issue #23517: Try to fix test_time on "x86 Ubuntu Shared 3.x" buildbot --- Python/pytime.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 8a2579b..ffb390a 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -64,11 +64,13 @@ _PyLong_FromTime_t(time_t t) static double _PyTime_RoundHalfUp(double x) { - if (x >= 0.0) - x = floor(x + 0.5); + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d = x; + if (d >= 0.0) + d = floor(d + 0.5); else - x = ceil(x - 0.5); - return x; + d = ceil(d - 0.5); + return d; } @@ -77,7 +79,7 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) { double intpart, err; - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + /* volatile avoids optimization changing how numbers are rounded */ volatile double floatpart; floatpart = modf(d, &intpart); @@ -134,7 +136,8 @@ int _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) { if (PyFloat_Check(obj)) { - double d, intpart, err; + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d, intpart, err; d = PyFloat_AsDouble(obj); if (round == _PyTime_ROUND_HALF_UP) @@ -255,7 +258,7 @@ static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, long to_nanoseconds) { - /* volatile avoids unsafe optimization on float enabled by gcc -O3 */ + /* volatile avoids optimization changing how numbers are rounded */ volatile double d, err; /* convert to a number of nanoseconds */ -- cgit v0.12 From 8aad8d6ad32de1f8fb87ff55ecde8ea3de8a72c4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 13:54:28 +0200 Subject: Issue #23517: test_time, skip a test checking a corner case on floating point rounding --- Lib/test/test_time.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index db96277..5947f40 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -825,7 +825,9 @@ class TestPyTime_t(unittest.TestCase): # close to 2^23 seconds (2**23 - 1e-9, 8388607999999999, FLOOR), (2**23 - 1e-9, 8388607999999999, CEILING), - (2**23 - 1e-9, 8388608000000000, HALF_UP), + # Issue #23517: skip HALF_UP test because the result is different + # depending on the FPU and how the compiler optimize the code :-/ + #(2**23 - 1e-9, 8388608000000000, HALF_UP), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) -- cgit v0.12 From f08fea9ee8a4ff12445b994dc7956b58e85762f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 14:23:40 +0200 Subject: Issue 24297: Fix test_symbol on Windows Don't rely on end of line. Open files in text mode, not in binary mode. --- Lib/test/test_symbol.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_symbol.py b/Lib/test/test_symbol.py index c126be3..2dcb9de 100644 --- a/Lib/test/test_symbol.py +++ b/Lib/test/test_symbol.py @@ -15,12 +15,11 @@ TEST_PY_FILE = 'symbol_test.py' class TestSymbolGeneration(unittest.TestCase): def _copy_file_without_generated_symbols(self, source_file, dest_file): - with open(source_file, 'rb') as fp: + with open(source_file) as fp: lines = fp.readlines() - nl = lines[0][len(lines[0].rstrip()):] - with open(dest_file, 'wb') as fp: - fp.writelines(lines[:lines.index(b"#--start constants--" + nl) + 1]) - fp.writelines(lines[lines.index(b"#--end constants--" + nl):]) + with open(dest_file, 'w') as fp: + fp.writelines(lines[:lines.index("#--start constants--\n") + 1]) + fp.writelines(lines[lines.index("#--end constants--\n"):]) def _generate_symbols(self, grammar_file, target_symbol_py_file): proc = subprocess.Popen([sys.executable, @@ -30,18 +29,26 @@ class TestSymbolGeneration(unittest.TestCase): stderr = proc.communicate()[1] return proc.returncode, stderr + def compare_files(self, file1, file2): + with open(file1) as fp: + lines1 = fp.readlines() + with open(file2) as fp: + lines2 = fp.readlines() + self.assertEqual(lines1, lines2) + @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), 'test only works from source build directory') def test_real_grammar_and_symbol_file(self): - self._copy_file_without_generated_symbols(SYMBOL_FILE, TEST_PY_FILE) - self.addCleanup(support.unlink, TEST_PY_FILE) - self.assertFalse(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE)) - self.assertEqual((0, b''), self._generate_symbols(GRAMMAR_FILE, - TEST_PY_FILE)) - self.assertTrue(filecmp.cmp(SYMBOL_FILE, TEST_PY_FILE), - 'symbol stat: %r\ntest_py stat: %r\n' % - (os.stat(SYMBOL_FILE), - os.stat(TEST_PY_FILE))) + output = support.TESTFN + self.addCleanup(support.unlink, output) + + self._copy_file_without_generated_symbols(SYMBOL_FILE, output) + + exitcode, stderr = self._generate_symbols(GRAMMAR_FILE, output) + self.assertEqual(b'', stderr) + self.assertEqual(0, exitcode) + + self.compare_files(SYMBOL_FILE, output) if __name__ == "__main__": -- cgit v0.12 From 6dad8f89623e588f486d069b7a336c1430c3c147 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:44:22 +0200 Subject: test_gdb: add debug info to investigate failure on "s390x SLES 3.x" buildbot --- Lib/test/test_gdb.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 0322677..193c97a 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -28,9 +28,13 @@ except OSError: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. raise unittest.SkipTest("Couldn't find gdb on the path") -gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) -gdb_major_version = int(gdb_version_number.group(1)) -gdb_minor_version = int(gdb_version_number.group(2)) +try: + gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) + gdb_major_version = int(gdb_version_number.group(1)) + gdb_minor_version = int(gdb_version_number.group(2)) +except Exception: + raise ValueError("unable to parse GDB version: %r" % gdb_version) + if gdb_major_version < 7: raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding" " Saw:\n" + gdb_version.decode('ascii', 'replace')) -- cgit v0.12 From e801c36037bf3875de0ecbb79c9f9ba8938027da Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 15:46:00 +0200 Subject: test_gdb: fix ResourceWarning if the test is interrupted --- Lib/test/test_gdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 193c97a..a51b552 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -63,9 +63,11 @@ def run_gdb(*args, **env_vars): base_cmd = ('gdb', '--batch', '-nx') if (gdb_major_version, gdb_minor_version) >= (7, 4): base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path) - out, err = subprocess.Popen(base_cmd + args, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, - ).communicate() + proc = subprocess.Popen(base_cmd + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + with proc: + out, err = proc.communicate() return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') # Verify that "gdb" was built with the embedded python support enabled: -- cgit v0.12 From 177b8eb34fed9a299ad00cea8e8be45fd836ab91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 17:19:04 +0200 Subject: test_eintr: try to debug hang on FreeBSD --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index f755880..73711a4 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,10 +37,17 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): -- cgit v0.12 From 2ec558739e6bd32365e1a883889f9d5372b35719 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 2 Sep 2015 19:16:07 +0200 Subject: Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN). --- Include/pytime.h | 4 ++++ Lib/datetime.py | 12 ++++++++++-- Lib/test/datetimetester.py | 14 +++++--------- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 22 +--------------------- Python/pytime.c | 3 +-- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 98ae12b..41fb806 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -44,6 +44,10 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); +/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ +PyAPI_FUNC(double) _PyTime_RoundHalfUp( + double x); + /* Convert a number of seconds, int or float, to time_t. */ PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, diff --git a/Lib/datetime.py b/Lib/datetime.py index db13b12..d661460 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,6 +316,14 @@ def _divide_and_round(a, b): return q +def _round_half_up(x): + """Round to nearest with ties going away from zero.""" + if x >= 0.0: + return _math.floor(x + 0.5) + else: + return _math.ceil(x - 0.5) + + class timedelta: """Represent the difference between two datetime objects. @@ -399,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -410,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = round(microseconds + usdouble) + microseconds = _round_half_up(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index babeb44..62f5527 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -662,28 +662,24 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=0)) - eq(td(milliseconds=-0.5/1000), td(microseconds=0)) + eq(td(milliseconds=0.5/1000), td(microseconds=1)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=0)) - eq(td(seconds=-0.5/10**6), td(microseconds=0)) + eq(td(seconds=0.5/10**6), td(microseconds=1)) + eq(td(seconds=-0.5/10**6), td(microseconds=-1)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) - # Test for a patch in Issue 8860 - eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) - eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) - def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), diff --git a/Misc/NEWS b/Misc/NEWS index e08905c..c9b925a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: datetime.timedelta constructor now rounds microseconds to + nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and + Python older than 3.3, instead of rounding to nearest with ties going to + nearest even integer (ROUND_HALF_EVEN). + - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 5cff3f8..6cab1e2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2149,29 +2149,9 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (leftover_us) { /* Round to nearest whole # of us, and add into x. */ double whole_us = round(leftover_us); - int x_is_odd; PyObject *temp; - whole_us = round(leftover_us); - if (fabs(whole_us - leftover_us) == 0.5) { - /* We're exactly halfway between two integers. In order - * to do round-half-to-even, we must determine whether x - * is odd. Note that x is odd when it's last bit is 1. The - * code below uses bitwise and operation to check the last - * bit. */ - temp = PyNumber_And(x, one); /* temp <- x & 1 */ - if (temp == NULL) { - Py_DECREF(x); - goto Done; - } - x_is_odd = PyObject_IsTrue(temp); - Py_DECREF(temp); - if (x_is_odd == -1) { - Py_DECREF(x); - goto Done; - } - whole_us = 2.0 * round((leftover_us + x_is_odd) * 0.5) - x_is_odd; - } + whole_us = _PyTime_RoundHalfUp(leftover_us); temp = PyLong_FromLong((long)whole_us); diff --git a/Python/pytime.c b/Python/pytime.c index ffb390a..02a1edf 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,8 +60,7 @@ _PyLong_FromTime_t(time_t t) #endif } -/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ -static double +double _PyTime_RoundHalfUp(double x) { /* volatile avoids optimization changing how numbers are rounded */ -- cgit v0.12 From be923ac948635e1a93d55398648f8d03996d3ebd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 01:38:44 +0200 Subject: Rewrite eintr_tester.py to avoid os.fork() eintr_tester.py calls signal.setitimer() to send signals to the current process every 100 ms. The test sometimes hangs on FreeBSD. Maybe there is a race condition in the child process after fork(). It's unsafe to run arbitrary code after fork(). This change replace os.fork() with a regular call to subprocess.Popen(). This change avoids the risk of having a child process which continue to execute eintr_tester.py instead of exiting. It also ensures that the child process doesn't inherit unexpected file descriptors by mistake. Since I'm unable to reproduce the issue on FreeBSD, I will have to watch FreeBSD buildbots to check if the issue is fixed or not. Remove previous attempt to debug: remove call to faulthandler.dump_traceback_later(). --- Lib/test/eintrdata/eintr_tester.py | 259 ++++++++++++++++++++++--------------- 1 file changed, 158 insertions(+), 101 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 73711a4..0616df6 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,12 +8,13 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ -import faulthandler import io import os import select import signal import socket +import subprocess +import sys import time import unittest @@ -29,7 +30,7 @@ class EINTRBaseTest(unittest.TestCase): # signal delivery periodicity signal_period = 0.1 # default sleep time for tests - should obviously have: - # sleep_time > signal_period + # sleep_time > signal_period sleep_time = 0.2 @classmethod @@ -37,17 +38,10 @@ class EINTRBaseTest(unittest.TestCase): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(15 * 60, exit=True) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -59,18 +53,22 @@ class EINTRBaseTest(unittest.TestCase): # default sleep time time.sleep(cls.sleep_time) + def subprocess(self, *args, **kw): + cmd_args = (sys.executable, '-c') + args + return subprocess.Popen(cmd_args, **kw) + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class OSEINTRTest(EINTRBaseTest): """ EINTR tests for the os module. """ + def new_sleep_process(self): + code = 'import time; time.sleep(%r)' % self.sleep_time + return self.subprocess(code) + def _test_wait_multiple(self, wait_func): num = 3 - for _ in range(num): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) + processes = [self.new_sleep_process() for _ in range(num)] for _ in range(num): wait_func() @@ -82,12 +80,8 @@ class OSEINTRTest(EINTRBaseTest): self._test_wait_multiple(lambda: os.wait3(0)) def _test_wait_single(self, wait_func): - pid = os.fork() - if pid == 0: - self._sleep() - os._exit(0) - else: - wait_func(pid) + proc = self.new_sleep_process() + wait_func(proc.pid) def test_waitpid(self): self._test_wait_single(lambda pid: os.waitpid(pid, 0)) @@ -105,19 +99,24 @@ class OSEINTRTest(EINTRBaseTest): # atomic datas = [b"hello", b"world", b"spam"] - pid = os.fork() - if pid == 0: - os.close(rd) - for data in datas: - # let the parent block on read() - self._sleep() - os.write(wr, data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, sys, time', + '', + 'wr = int(sys.argv[1])', + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'for data in datas:', + ' # let the parent block on read()', + ' time.sleep(sleep_time)', + ' os.write(wr, data)', + )) + + with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_write(self): rd, wr = os.pipe() @@ -127,23 +126,34 @@ class OSEINTRTest(EINTRBaseTest): # we must write enough data for the write() to block data = b"xyz" * support.PIPE_MAX_SIZE - pid = os.fork() - if pid == 0: - os.close(wr) - read_data = io.BytesIO() - # let the parent block on write() - self._sleep() - while len(read_data.getvalue()) < len(data): - chunk = os.read(rd, 2 * len(data)) - read_data.write(chunk) - self.assertEqual(read_data.getvalue(), data) - os._exit(0) - else: + code = '\n'.join(( + 'import io, os, sys, time', + '', + 'rd = int(sys.argv[1])', + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data_len = len(data)', + '', + '# let the parent block on write()', + 'time.sleep(sleep_time)', + '', + 'read_data = io.BytesIO()', + 'while len(read_data.getvalue()) < data_len:', + ' chunk = os.read(rd, 2 * data_len)', + ' read_data.write(chunk)', + '', + 'value = read_data.getvalue()', + 'if value != data:', + ' raise Exception("read error: %s vs %s bytes"', + ' % (len(value), data_len))', + )) + + with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: os.close(rd) written = 0 while written < len(data): written += os.write(wr, memoryview(data)[written:]) - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") @@ -159,19 +169,32 @@ class SocketEINTRTest(EINTRBaseTest): # single-byte payload guard us against partial recv datas = [b"x", b"y", b"z"] - pid = os.fork() - if pid == 0: - rd.close() - for data in datas: - # let the parent block on recv() - self._sleep() - wr.sendall(data) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(wr.family), + 'sock_type = %s' % int(wr.type), + 'datas = %r' % datas, + 'sleep_time = %r' % self.sleep_time, + '', + 'wr = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with wr:', + ' for data in datas:', + ' # let the parent block on recv()', + ' time.sleep(sleep_time)', + ' wr.sendall(data)', + )) + + fd = wr.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) + self.assertEqual(proc.wait(), 0) def test_recv(self): self._test_recv(socket.socket.recv) @@ -188,25 +211,43 @@ class SocketEINTRTest(EINTRBaseTest): # we must send enough data for the send() to block data = b"xyz" * (support.SOCK_MAX_SIZE // 3) - pid = os.fork() - if pid == 0: - wr.close() - # let the parent block on send() - self._sleep() - received_data = bytearray(len(data)) - n = 0 - while n < len(data): - n += rd.recv_into(memoryview(received_data)[n:]) - self.assertEqual(received_data, data) - os._exit(0) - else: + code = '\n'.join(( + 'import os, socket, sys, time', + '', + 'fd = int(sys.argv[1])', + 'family = %s' % int(rd.family), + 'sock_type = %s' % int(rd.type), + 'sleep_time = %r' % self.sleep_time, + 'data = b"xyz" * %s' % (support.SOCK_MAX_SIZE // 3), + 'data_len = len(data)', + '', + 'rd = socket.fromfd(fd, family, sock_type)', + 'os.close(fd)', + '', + 'with rd:', + ' # let the parent block on send()', + ' time.sleep(sleep_time)', + '', + ' received_data = bytearray(data_len)', + ' n = 0', + ' while n < data_len:', + ' n += rd.recv_into(memoryview(received_data)[n:])', + '', + 'if received_data != data:', + ' raise Exception("recv error: %s vs %s bytes"', + ' % (len(received_data), data_len))', + )) + + fd = rd.fileno() + proc = self.subprocess(code, str(fd), pass_fds=[fd]) + with proc: rd.close() written = 0 while written < len(data): sent = send_func(wr, memoryview(data)[written:]) # sendall() returns None written += len(data) if sent is None else sent - self.assertEqual(0, os.waitpid(pid, 0)[1]) + self.assertEqual(proc.wait(), 0) def test_send(self): self._test_send(socket.socket.send) @@ -223,45 +264,60 @@ class SocketEINTRTest(EINTRBaseTest): self.addCleanup(sock.close) sock.bind((support.HOST, 0)) - _, port = sock.getsockname() + port = sock.getsockname()[1] sock.listen() - pid = os.fork() - if pid == 0: - # let parent block on accept() - self._sleep() - with socket.create_connection((support.HOST, port)): - self._sleep() - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) + code = '\n'.join(( + 'import socket, time', + '', + 'host = %r' % support.HOST, + 'port = %s' % port, + 'sleep_time = %r' % self.sleep_time, + '', + '# let parent block on accept()', + 'time.sleep(sleep_time)', + 'with socket.create_connection((host, port)):', + ' time.sleep(sleep_time)', + )) + + with self.subprocess(code) as proc: client_sock, _ = sock.accept() client_sock.close() + self.assertEqual(proc.wait(), 0) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): + filename = support.TESTFN + # Use a fifo: until the child opens it for reading, the parent will # block when trying to open it for writing. - support.unlink(support.TESTFN) - os.mkfifo(support.TESTFN) - self.addCleanup(support.unlink, support.TESTFN) - - pid = os.fork() - if pid == 0: - # let the parent block - self._sleep() - do_open_close_reader(support.TESTFN) - os._exit(0) - else: - self.addCleanup(os.waitpid, pid, 0) - do_open_close_writer(support.TESTFN) + support.unlink(filename) + os.mkfifo(filename) + self.addCleanup(support.unlink, filename) + + code = '\n'.join(( + 'import os, time', + '', + 'path = %a' % filename, + 'sleep_time = %r' % self.sleep_time, + '', + '# let the parent block', + 'time.sleep(sleep_time)', + '', + do_open_close_reader, + )) + + with self.subprocess(code) as proc: + do_open_close_writer(filename) + + self.assertEqual(proc.wait(), 0) def test_open(self): - self._test_open(lambda path: open(path, 'r').close(), + self._test_open("open(path, 'r').close()", lambda path: open(path, 'w').close()) def test_os_open(self): - self._test_open(lambda path: os.close(os.open(path, os.O_RDONLY)), + self._test_open("os.close(os.open(path, os.O_RDONLY))", lambda path: os.close(os.open(path, os.O_WRONLY))) @@ -298,20 +354,21 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + code = '\n'.join(( + 'import os, time', + 'pid = %s' % os.getpid(), + 'signum = %s' % int(signum), + 'sleep_time = %r' % self.sleep_time, + 'time.sleep(sleep_time)', + 'os.kill(pid, signum)', + )) + t0 = time.monotonic() - child_pid = os.fork() - if child_pid == 0: - # child - try: - self._sleep() - os.kill(pid, signum) - finally: - os._exit(0) - else: + with self.subprocess(code) as proc: # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 - os.waitpid(child_pid, 0) + self.assertEqual(proc.wait(), 0) self.assertGreaterEqual(dt, self.sleep_time) -- cgit v0.12 From 2ec5bd6fb22ff71ffbe5987f55a31bc4177e39a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 09:06:44 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding towards -Infinity (ROUND_FLOOR). --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 11 ++++++----- Misc/NEWS | 5 +++++ Modules/_datetimemodule.c | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index d661460..5ba2ddb 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1384,7 +1384,7 @@ class datetime(date): converter = _time.localtime if tz is None else _time.gmtime t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, @@ -1404,7 +1404,7 @@ class datetime(date): def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" t, frac = divmod(t, 1.0) - us = int(frac * 1e6) + us = _round_half_up(frac * 1e6) # If timestamp is less than one microsecond smaller than a # full second, us can be rounded up to 1000000. In this case, diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 62f5527..f516434 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1847,6 +1847,7 @@ class TestDateTime(TestDate): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) + one = fts(1e-6) try: minus_one = fts(-1e-6) except OSError: @@ -1857,22 +1858,22 @@ class TestDateTime(TestDate): self.assertEqual(minus_one.microsecond, 999999) t = fts(-1e-8) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(-9e-7) self.assertEqual(t, minus_one) t = fts(-1e-7) - self.assertEqual(t, minus_one) + self.assertEqual(t, zero) t = fts(1e-7) self.assertEqual(t, zero) t = fts(9e-7) - self.assertEqual(t, zero) + self.assertEqual(t, one) t = fts(0.99999949) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 999999) t = fts(0.9999999) - self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 999999) + self.assertEqual(t.second, 1) + self.assertEqual(t.microsecond, 0) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Misc/NEWS b/Misc/NEWS index c9b925a..9af32ad 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,11 @@ Core and Builtins Library ------- +- Issue #23517: fromtimestamp() and utcfromtimestamp() methods of + datetime.datetime now round microseconds to nearest with ties going away from + zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of + rounding towards -Infinity (ROUND_FLOOR). + - Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 6cab1e2..ae459df 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4078,7 +4078,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_FLOOR) == -1) + &timet, &us, _PyTime_ROUND_HALF_UP) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); -- cgit v0.12 From 99bb14bf0c4b9bdef2535710cc14abbdf5aa648a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 12:16:49 +0200 Subject: type_call() now detect bugs in type new and init * Call _Py_CheckFunctionResult() to check for bugs in type constructors (tp_new) * Add assertions to ensure an exception was raised if tp_init failed or that no exception was raised if tp_init succeed Refactor also the function to have less indentation. --- Objects/typeobject.c | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 1beed72..10115ef 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -906,25 +906,33 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif obj = type->tp_new(type, args, kwds); - if (obj != NULL) { - /* Ugly exception: when the call was type(something), - don't call tp_init on the result. */ - if (type == &PyType_Type && - PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1 && - (kwds == NULL || - (PyDict_Check(kwds) && PyDict_Size(kwds) == 0))) - return obj; - /* If the returned object is not an instance of type, - it won't be initialized. */ - if (!PyType_IsSubtype(Py_TYPE(obj), type)) - return obj; - type = Py_TYPE(obj); - if (type->tp_init != NULL) { - int res = type->tp_init(obj, args, kwds); - if (res < 0) { - Py_DECREF(obj); - obj = NULL; - } + obj = _Py_CheckFunctionResult((PyObject*)type, obj, NULL); + if (obj == NULL) + return NULL; + + /* Ugly exception: when the call was type(something), + don't call tp_init on the result. */ + if (type == &PyType_Type && + PyTuple_Check(args) && PyTuple_GET_SIZE(args) == 1 && + (kwds == NULL || + (PyDict_Check(kwds) && PyDict_Size(kwds) == 0))) + return obj; + + /* If the returned object is not an instance of type, + it won't be initialized. */ + if (!PyType_IsSubtype(Py_TYPE(obj), type)) + return obj; + + type = Py_TYPE(obj); + if (type->tp_init != NULL) { + int res = type->tp_init(obj, args, kwds); + if (res < 0) { + assert(PyErr_Occurred()); + Py_DECREF(obj); + obj = NULL; + } + else { + assert(!PyErr_Occurred()); } } return obj; -- cgit v0.12 From 00723e03536c8494e692767f8495800188843d6f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 12:57:11 +0200 Subject: Fix ast_for_atom() Clear PyObject_Str() exception if it failed, ast_error() should not be called with an exception set. --- Python/ast.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ast.c b/Python/ast.c index c1ce0aa..1f7ddfc 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -2040,6 +2040,7 @@ ast_for_atom(struct compiling *c, const node *n) PyOS_snprintf(buf, sizeof(buf), "(%s) %s", errtype, s); Py_DECREF(errstr); } else { + PyErr_Clear(); PyOS_snprintf(buf, sizeof(buf), "(%s) unknown error", errtype); } ast_error(c, n, buf); -- cgit v0.12 From 29ee6745af8ace80cef04c18b232771e8b00acdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 16:25:45 +0200 Subject: Enhance _PyTime_AsTimespec() Ensure that the tv_nsec field is set, even if the function fails with an overflow. --- Python/pytime.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 8166cec..cea3cf5 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -479,13 +479,13 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) secs -= 1; } ts->tv_sec = (time_t)secs; + assert(0 <= nsec && nsec < SEC_TO_NS); + ts->tv_nsec = nsec; + if ((_PyTime_t)ts->tv_sec != secs) { _PyTime_overflow(); return -1; } - ts->tv_nsec = nsec; - - assert(0 <= ts->tv_nsec && ts->tv_nsec < SEC_TO_NS); return 0; } #endif -- cgit v0.12 From 5786aef382b0f64720060d6372c215e939461957 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 3 Sep 2015 16:33:16 +0200 Subject: Don't abuse volatile keyword in pytime.c Only use it on the most important number. This change fixes also a compiler warning on modf(). --- Python/pytime.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index cea3cf5..d226389 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -135,8 +135,9 @@ int _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) { if (PyFloat_Check(obj)) { + double intpart, err; /* volatile avoids optimization changing how numbers are rounded */ - volatile double d, intpart, err; + volatile double d; d = PyFloat_AsDouble(obj); if (round == _PyTime_ROUND_HALF_UP) @@ -257,8 +258,9 @@ static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, long to_nanoseconds) { + double err; /* volatile avoids optimization changing how numbers are rounded */ - volatile double d, err; + volatile double d; /* convert to a number of nanoseconds */ d = value; -- cgit v0.12 From 110f9e35383373022b169756e64974814733aef4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 10:31:16 +0200 Subject: test_time: add tests on HALF_UP rounding mode for _PyTime_ObjectToTime_t() and _PyTime_ObjectToTimespec() --- Lib/test/test_time.py | 130 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 40 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 5947f40..590425a 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -606,24 +606,49 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_time_t(self): from _testcapi import pytime_object_to_time_t + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, seconds in ( + # int + (-1, -1), + (0, 0), + (1, 1), + + # float + (-1.0, -1), + (1.0, 1), + ): + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), + seconds) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, time_t, rnd in ( - # Round towards minus infinity (-inf) - (0, 0, _PyTime.ROUND_FLOOR), - (-1, -1, _PyTime.ROUND_FLOOR), - (-1.0, -1, _PyTime.ROUND_FLOOR), - (-1.9, -2, _PyTime.ROUND_FLOOR), - (1.0, 1, _PyTime.ROUND_FLOOR), - (1.9, 1, _PyTime.ROUND_FLOOR), - # Round towards infinity (+inf) - (0, 0, _PyTime.ROUND_CEILING), - (-1, -1, _PyTime.ROUND_CEILING), - (-1.0, -1, _PyTime.ROUND_CEILING), - (-1.9, -1, _PyTime.ROUND_CEILING), - (1.0, 1, _PyTime.ROUND_CEILING), - (1.9, 2, _PyTime.ROUND_CEILING), + (-1.9, -2, FLOOR), + (-1.9, -1, CEILING), + (-1.9, -2, HALF_UP), + + (1.9, 1, FLOOR), + (1.9, 2, CEILING), + (1.9, 2, HALF_UP), + + # half up + (-0.999, -1, HALF_UP), + (-0.510, -1, HALF_UP), + (-0.500, -1, HALF_UP), + (-0.490, 0, HALF_UP), + ( 0.490, 0, HALF_UP), + ( 0.500, 1, HALF_UP), + ( 0.510, 1, HALF_UP), + ( 0.999, 1, HALF_UP), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, @@ -632,39 +657,64 @@ class TestPytime(unittest.TestCase): @support.cpython_only def test_timespec(self): from _testcapi import pytime_object_to_timespec + + # Conversion giving the same result for all rounding methods + for rnd in ALL_ROUNDING_METHODS: + for obj, timespec in ( + # int + (0, (0, 0)), + (-1, (-1, 0)), + + # float + (-1.2, (-2, 800000000)), + (-1.0, (-1, 0)), + (-1e-9, (-1, 999999999)), + (1e-9, (0, 1)), + (1.0, (1, 0)), + ): + with self.subTest(obj=obj, round=rnd, timespec=timespec): + self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + + # Conversion giving different results depending on the rounding method + FLOOR = _PyTime.ROUND_FLOOR + CEILING = _PyTime.ROUND_CEILING + HALF_UP = _PyTime.ROUND_HALF_UP for obj, timespec, rnd in ( # Round towards minus infinity (-inf) - (0, (0, 0), _PyTime.ROUND_FLOOR), - (-1, (-1, 0), _PyTime.ROUND_FLOOR), - (-1.0, (-1, 0), _PyTime.ROUND_FLOOR), - (1e-9, (0, 1), _PyTime.ROUND_FLOOR), - (1e-10, (0, 0), _PyTime.ROUND_FLOOR), - (-1e-9, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1e-10, (-1, 999999999), _PyTime.ROUND_FLOOR), - (-1.2, (-2, 800000000), _PyTime.ROUND_FLOOR), - (0.9999999999, (0, 999999999), _PyTime.ROUND_FLOOR), - (1.1234567890, (1, 123456789), _PyTime.ROUND_FLOOR), - (1.1234567899, (1, 123456789), _PyTime.ROUND_FLOOR), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_FLOOR), - (-1.1234567891, (-2, 876543210), _PyTime.ROUND_FLOOR), + (-1e-10, (0, 0), CEILING), + (-1e-10, (-1, 999999999), FLOOR), + (-1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), FLOOR), + (1e-10, (0, 1), CEILING), + (1e-10, (0, 0), HALF_UP), + + (0.9999999999, (0, 999999999), FLOOR), + (0.9999999999, (1, 0), CEILING), + + (1.1234567890, (1, 123456789), FLOOR), + (1.1234567899, (1, 123456789), FLOOR), + (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) - (0, (0, 0), _PyTime.ROUND_CEILING), - (-1, (-1, 0), _PyTime.ROUND_CEILING), - (-1.0, (-1, 0), _PyTime.ROUND_CEILING), - (1e-9, (0, 1), _PyTime.ROUND_CEILING), - (1e-10, (0, 1), _PyTime.ROUND_CEILING), - (-1e-9, (-1, 999999999), _PyTime.ROUND_CEILING), - (-1e-10, (0, 0), _PyTime.ROUND_CEILING), - (-1.2, (-2, 800000000), _PyTime.ROUND_CEILING), - (0.9999999999, (1, 0), _PyTime.ROUND_CEILING), - (1.1234567890, (1, 123456790), _PyTime.ROUND_CEILING), - (1.1234567899, (1, 123456790), _PyTime.ROUND_CEILING), - (-1.1234567890, (-2, 876543211), _PyTime.ROUND_CEILING), - (-1.1234567891, (-2, 876543211), _PyTime.ROUND_CEILING), + (1.1234567890, (1, 123456790), CEILING), + (1.1234567899, (1, 123456790), CEILING), + (-1.1234567890, (-2, 876543211), CEILING), + (-1.1234567891, (-2, 876543211), CEILING), + + # half up + (-0.6e-9, (-1, 999999999), HALF_UP), + # skipped, 0.5e-6 is inexact in base 2 + #(-0.5e-9, (-1, 999999999), HALF_UP), + (-0.4e-9, (0, 0), HALF_UP), + + (0.4e-9, (0, 0), HALF_UP), + (0.5e-9, (0, 1), HALF_UP), + (0.6e-9, (0, 1), HALF_UP), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) + # Test OverflowError rnd = _PyTime.ROUND_FLOOR for invalid in self.invalid_values: self.assertRaises(OverflowError, -- cgit v0.12 From adfefa527a32e711c1bea9c1ac32c20e9cce0660 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Sep 2015 23:57:25 +0200 Subject: Issue #23517: Fix implementation of the ROUND_HALF_UP rounding mode in datetime.datetime.fromtimestamp() and datetime.datetime.utcfromtimestamp(). microseconds sign should be kept before rounding. --- Lib/datetime.py | 48 ++++++++++++++++++++-------------------------- Lib/test/datetimetester.py | 12 ++++++++++-- Lib/test/test_time.py | 13 +++++++------ Python/pytime.c | 8 ++++---- 4 files changed, 42 insertions(+), 39 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 5ba2ddb..3bf9edc 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1374,28 +1374,34 @@ class datetime(date): return self._tzinfo @classmethod - def fromtimestamp(cls, t, tz=None): + def _fromtimestamp(cls, t, utc, tz): """Construct a datetime from a POSIX timestamp (like time.time()). A timezone info object may be passed in as well. """ - _check_tzinfo_arg(tz) - - converter = _time.localtime if tz is None else _time.gmtime - - t, frac = divmod(t, 1.0) + frac, t = _math.modf(t) us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: + if us >= 1000000: t += 1 - us = 0 + us -= 1000000 + elif us < 0: + t -= 1 + us += 1000000 + + converter = _time.gmtime if utc else _time.localtime y, m, d, hh, mm, ss, weekday, jday, dst = converter(t) ss = min(ss, 59) # clamp out leap seconds if the platform has them - result = cls(y, m, d, hh, mm, ss, us, tz) + return cls(y, m, d, hh, mm, ss, us, tz) + + @classmethod + def fromtimestamp(cls, t, tz=None): + """Construct a datetime from a POSIX timestamp (like time.time()). + + A timezone info object may be passed in as well. + """ + _check_tzinfo_arg(tz) + + result = cls._fromtimestamp(t, tz is not None, tz) if tz is not None: result = tz.fromutc(result) return result @@ -1403,19 +1409,7 @@ class datetime(date): @classmethod def utcfromtimestamp(cls, t): """Construct a naive UTC datetime from a POSIX timestamp.""" - t, frac = divmod(t, 1.0) - us = _round_half_up(frac * 1e6) - - # If timestamp is less than one microsecond smaller than a - # full second, us can be rounded up to 1000000. In this case, - # roll over to seconds, otherwise, ValueError is raised - # by the constructor. - if us == 1000000: - t += 1 - us = 0 - y, m, d, hh, mm, ss, weekday, jday, dst = _time.gmtime(t) - ss = min(ss, 59) # clamp out leap seconds if the platform has them - return cls(y, m, d, hh, mm, ss, us) + return cls._fromtimestamp(t, True, None) @classmethod def now(cls, tz=None): diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f516434..58873af 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -668,6 +668,8 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) eq(td(seconds=0.5/10**6), td(microseconds=1)) eq(td(seconds=-0.5/10**6), td(microseconds=-1)) + eq(td(seconds=1/2**7), td(microseconds=7813)) + eq(td(seconds=-1/2**7), td(microseconds=-7813)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -1842,8 +1844,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in [self.theclass.fromtimestamp, - self.theclass.utcfromtimestamp]: + for fts in (datetime.fromtimestamp, + self.theclass.utcfromtimestamp): zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) @@ -1874,6 +1876,12 @@ class TestDateTime(TestDate): t = fts(0.9999999) self.assertEqual(t.second, 1) self.assertEqual(t.microsecond, 0) + t = fts(1/2**7) + self.assertEqual(t.second, 0) + self.assertEqual(t.microsecond, 7813) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 590425a..75ab666 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -655,7 +655,7 @@ class TestPytime(unittest.TestCase): pytime_object_to_time_t, invalid, rnd) @support.cpython_only - def test_timespec(self): + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec # Conversion giving the same result for all rounding methods @@ -666,7 +666,7 @@ class TestPytime(unittest.TestCase): (-1, (-1, 0)), # float - (-1.2, (-2, 800000000)), + (-1/2**7, (-1, 992187500)), (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), @@ -693,7 +693,7 @@ class TestPytime(unittest.TestCase): (1.1234567890, (1, 123456789), FLOOR), (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543211), FLOOR), + (-1.1234567890, (-2, 876543210), FLOOR), (-1.1234567891, (-2, 876543210), FLOOR), # Round towards infinity (+inf) (1.1234567890, (1, 123456790), CEILING), @@ -1155,7 +1155,7 @@ class TestOldPyTime(unittest.TestCase): self.assertRaises(OverflowError, pytime_object_to_time_t, invalid, rnd) - def test_timeval(self): + def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval # Conversion giving the same result for all rounding methods @@ -1167,7 +1167,8 @@ class TestOldPyTime(unittest.TestCase): # float (-1.0, (-1, 0)), - (-1.2, (-2, 800000)), + (1/2**6, (0, 15625)), + (-1/2**6, (-1, 984375)), (-1e-6, (-1, 999999)), (1e-6, (0, 1)), ): @@ -1225,7 +1226,7 @@ class TestOldPyTime(unittest.TestCase): (-1.0, (-1, 0)), (-1e-9, (-1, 999999999)), (1e-9, (0, 1)), - (-1.2, (-2, 800000000)), + (-1/2**9, (-1, 998046875)), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), diff --git a/Python/pytime.c b/Python/pytime.c index d226389..0364473 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -82,10 +82,6 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, volatile double floatpart; floatpart = modf(d, &intpart); - if (floatpart < 0) { - floatpart += 1.0; - intpart -= 1.0; - } floatpart *= denominator; if (round == _PyTime_ROUND_HALF_UP) @@ -98,6 +94,10 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart -= denominator; intpart += 1.0; } + else if (floatpart < 0) { + floatpart += denominator; + intpart -= 1.0; + } assert(0.0 <= floatpart && floatpart < denominator); *sec = (time_t)intpart; -- cgit v0.12 From 8820a350d77c1cf93a6c720981e562407c2dd71d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 5 Sep 2015 10:50:20 +0200 Subject: Issue #23517: Skip a datetime test on Windows The test calls gmtime(-1)/localtime(-1) which is not supported on Windows. --- Lib/test/datetimetester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 58873af..b8c9b0d 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1865,6 +1865,9 @@ class TestDateTime(TestDate): self.assertEqual(t, minus_one) t = fts(-1e-7) self.assertEqual(t, zero) + t = fts(-1/2**7) + self.assertEqual(t.second, 59) + self.assertEqual(t.microsecond, 992187) t = fts(1e-7) self.assertEqual(t, zero) @@ -1879,9 +1882,6 @@ class TestDateTime(TestDate): t = fts(1/2**7) self.assertEqual(t.second, 0) self.assertEqual(t.microsecond, 7813) - t = fts(-1/2**7) - self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, -- cgit v0.12 From 7827a5b7c29ae71daf0175ce3398115374ceb50e Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 6 Sep 2015 13:07:21 -0400 Subject: Closes Issue#22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. --- Doc/library/datetime.rst | 19 ++++++++++++------- Lib/datetime.py | 2 ++ Lib/test/datetimetester.py | 3 ++- Misc/NEWS | 2 ++ Modules/_datetimemodule.c | 5 +++++ 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 976cd49..9a0f936 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1734,10 +1734,7 @@ made to civil time. otherwise :exc:`ValueError` is raised. The *name* argument is optional. If specified it must be a string that - is used as the value returned by the ``tzname(dt)`` method. Otherwise, - ``tzname(dt)`` returns a string 'UTCsHH:MM', where s is the sign of - *offset*, HH and MM are two digits of ``offset.hours`` and - ``offset.minutes`` respectively. + will be used as the value returned by the :meth:`datetime.tzname` method. .. versionadded:: 3.2 @@ -1750,11 +1747,19 @@ made to civil time. .. method:: timezone.tzname(dt) - Return the fixed value specified when the :class:`timezone` instance is - constructed or a string 'UTCsHH:MM', where s is the sign of - *offset*, HH and MM are two digits of ``offset.hours`` and + Return the fixed value specified when the :class:`timezone` instance + is constructed. If *name* is not provided in the constructor, the + name returned by ``tzname(dt)`` is generated from the value of the + ``offset`` as follows. If *offset* is ``timedelta(0)``, the name + is "UTC", otherwise it is a string 'UTC±HH:MM', where ± is the sign + of ``offset``, HH and MM are two digits of ``offset.hours`` and ``offset.minutes`` respectively. + .. versionchanged:: 3.6 + Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not + 'UTC+00:00'. + + .. method:: timezone.dst(dt) Always returns ``None``. diff --git a/Lib/datetime.py b/Lib/datetime.py index 3bf9edc..6b2ac33 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1920,6 +1920,8 @@ class timezone(tzinfo): @staticmethod def _name_from_offset(delta): + if not delta: + return 'UTC' if delta < timedelta(0): sign = '-' delta = -delta diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b8c9b0d..f23add3 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -258,7 +258,8 @@ class TestTimeZone(unittest.TestCase): with self.assertRaises(TypeError): self.EST.dst(5) def test_tzname(self): - self.assertEqual('UTC+00:00', timezone(ZERO).tzname(None)) + self.assertEqual('UTC', timezone.utc.tzname(None)) + self.assertEqual('UTC', timezone(ZERO).tzname(None)) self.assertEqual('UTC-05:00', timezone(-5 * HOUR).tzname(None)) self.assertEqual('UTC+09:30', timezone(9.5 * HOUR).tzname(None)) self.assertEqual('UTC-00:01', timezone(timedelta(minutes=-1)).tzname(None)) diff --git a/Misc/NEWS b/Misc/NEWS index e88106a..096ab67 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,6 +17,8 @@ Core and Builtins Library ------- +- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. + - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ae459df..008b733 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3267,6 +3267,11 @@ timezone_str(PyDateTime_TimeZone *self) Py_INCREF(self->name); return self->name; } + if (self == PyDateTime_TimeZone_UTC || + (GET_TD_DAYS(self->offset) == 0 && + GET_TD_SECONDS(self->offset) == 0 && + GET_TD_MICROSECONDS(self->offset) == 0)) + return PyUnicode_FromString("UTC"); /* Offset is normalized, so it is negative if days < 0 */ if (GET_TD_DAYS(self->offset) < 0) { sign = '-'; -- cgit v0.12 From 56f6e76c680f47ad2b11bed9406305a000a1889a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Sep 2015 21:25:30 +0300 Subject: Issue #15989: Fixed some scarcely probable integer overflows. It is very unlikely that they can occur in real code for now. --- Modules/_datetimemodule.c | 2 +- Modules/_io/_iomodule.c | 3 ++- Modules/posixmodule.c | 7 +++++-- Modules/readline.c | 2 +- Objects/structseq.c | 24 ++++++++++++------------ Python/Python-ast.c | 2 +- Python/pythonrun.c | 10 +++++----- 7 files changed, 27 insertions(+), 23 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 008b733..fe9a948 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4692,7 +4692,7 @@ local_timezone(PyDateTime_DateTime *utc_time) if (seconds == NULL) goto error; Py_DECREF(delta); - timestamp = PyLong_AsLong(seconds); + timestamp = _PyLong_AsTime_t(seconds); Py_DECREF(seconds); if (timestamp == -1 && PyErr_Occurred()) return NULL; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 1c2d3a0..7428aed 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -238,7 +238,8 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, int text = 0, binary = 0, universal = 0; char rawmode[6], *m; - int line_buffering, isatty; + int line_buffering; + long isatty; PyObject *raw, *modeobj = NULL, *buffer, *wrapper, *result = NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index d2b8dfd..270de0f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9481,7 +9481,7 @@ os__getdiskusage_impl(PyModuleDef *module, Py_UNICODE *path) */ struct constdef { char *name; - long value; + int value; }; static int @@ -9489,7 +9489,10 @@ conv_confname(PyObject *arg, int *valuep, struct constdef *table, size_t tablesize) { if (PyLong_Check(arg)) { - *valuep = PyLong_AS_LONG(arg); + int value = _PyLong_AsInt(arg); + if (value == -1 && PyErr_Occurred()) + return 0; + *valuep = value; return 1; } else { diff --git a/Modules/readline.c b/Modules/readline.c index f6b52a0..09877f2 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -840,7 +840,7 @@ on_hook(PyObject *func) if (r == Py_None) result = 0; else { - result = PyLong_AsLong(r); + result = _PyLong_AsInt(r); if (result == -1 && PyErr_Occurred()) goto error; } diff --git a/Objects/structseq.c b/Objects/structseq.c index 664344b..7209738 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -16,14 +16,14 @@ _Py_IDENTIFIER(n_fields); _Py_IDENTIFIER(n_unnamed_fields); #define VISIBLE_SIZE(op) Py_SIZE(op) -#define VISIBLE_SIZE_TP(tp) PyLong_AsLong( \ +#define VISIBLE_SIZE_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_sequence_fields)) -#define REAL_SIZE_TP(tp) PyLong_AsLong( \ +#define REAL_SIZE_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_fields)) #define REAL_SIZE(op) REAL_SIZE_TP(Py_TYPE(op)) -#define UNNAMED_FIELDS_TP(tp) PyLong_AsLong( \ +#define UNNAMED_FIELDS_TP(tp) PyLong_AsSsize_t( \ _PyDict_GetItemId((tp)->tp_dict, &PyId_n_unnamed_fields)) #define UNNAMED_FIELDS(op) UNNAMED_FIELDS_TP(Py_TYPE(op)) @@ -164,7 +164,8 @@ structseq_repr(PyStructSequence *obj) #define TYPE_MAXSIZE 100 PyTypeObject *typ = Py_TYPE(obj); - int i, removelast = 0; + Py_ssize_t i; + int removelast = 0; Py_ssize_t len; char buf[REPR_BUFFER_SIZE]; char *endofbuf, *pbuf = buf; @@ -236,8 +237,7 @@ structseq_reduce(PyStructSequence* self) PyObject* tup = NULL; PyObject* dict = NULL; PyObject* result; - Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields; - int i; + Py_ssize_t n_fields, n_visible_fields, n_unnamed_fields, i; n_fields = REAL_SIZE(self); n_visible_fields = VISIBLE_SIZE(self); @@ -325,7 +325,7 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) { PyObject *dict; PyMemberDef* members; - int n_members, n_unnamed_members, i, k; + Py_ssize_t n_members, n_unnamed_members, i, k; PyObject *v; #ifdef Py_TRACE_REFS @@ -373,9 +373,9 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) Py_INCREF(type); dict = type->tp_dict; -#define SET_DICT_FROM_INT(key, value) \ +#define SET_DICT_FROM_SIZE(key, value) \ do { \ - v = PyLong_FromLong((long) value); \ + v = PyLong_FromSsize_t(value); \ if (v == NULL) \ return -1; \ if (PyDict_SetItemString(dict, key, v) < 0) { \ @@ -385,9 +385,9 @@ PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) Py_DECREF(v); \ } while (0) - SET_DICT_FROM_INT(visible_length_key, desc->n_in_sequence); - SET_DICT_FROM_INT(real_length_key, n_members); - SET_DICT_FROM_INT(unnamed_fields_key, n_unnamed_members); + SET_DICT_FROM_SIZE(visible_length_key, desc->n_in_sequence); + SET_DICT_FROM_SIZE(real_length_key, n_members); + SET_DICT_FROM_SIZE(unnamed_fields_key, n_unnamed_members); return 0; } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 8a2dc7c..fd7f17e 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -769,7 +769,7 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) return 1; } - i = (int)PyLong_AsLong(obj); + i = _PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index ebedd12..1a5dab5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -431,7 +431,7 @@ static int parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, int *lineno, int *offset, PyObject **text) { - long hold; + int hold; PyObject *v; _Py_IDENTIFIER(msg); _Py_IDENTIFIER(filename); @@ -464,11 +464,11 @@ parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, v = _PyObject_GetAttrId(err, &PyId_lineno); if (!v) goto finally; - hold = PyLong_AsLong(v); + hold = _PyLong_AsInt(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; - *lineno = (int)hold; + *lineno = hold; v = _PyObject_GetAttrId(err, &PyId_offset); if (!v) @@ -477,11 +477,11 @@ parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename, *offset = -1; Py_DECREF(v); } else { - hold = PyLong_AsLong(v); + hold = _PyLong_AsInt(v); Py_DECREF(v); if (hold < 0 && PyErr_Occurred()) goto finally; - *offset = (int)hold; + *offset = hold; } v = _PyObject_GetAttrId(err, &PyId_text); -- cgit v0.12 From 481d3af82e4fe507a995ade313c68d6b635e8757 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Sep 2015 23:29:04 +0300 Subject: Make asdl_c.py to generate Python-ast.c changed in issue #15989. --- Parser/asdl_c.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a5e35d9..d597122 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -898,7 +898,7 @@ static int obj2ast_int(PyObject* obj, int* out, PyArena* arena) return 1; } - i = (int)PyLong_AsLong(obj); + i = _PyLong_AsInt(obj); if (i == -1 && PyErr_Occurred()) return 1; *out = i; -- cgit v0.12 From 90fd895197f485583af630ef2e34cbdea298abf7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Sep 2015 00:12:49 +0200 Subject: Issue #22241: Fix a compiler waring --- Modules/_datetimemodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index fe9a948..ea2bae6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3267,7 +3267,7 @@ timezone_str(PyDateTime_TimeZone *self) Py_INCREF(self->name); return self->name; } - if (self == PyDateTime_TimeZone_UTC || + if ((PyObject *)self == PyDateTime_TimeZone_UTC || (GET_TD_DAYS(self->offset) == 0 && GET_TD_SECONDS(self->offset) == 0 && GET_TD_MICROSECONDS(self->offset) == 0)) -- cgit v0.12 From 69cc487df42d9064a74551ae26a8c115dade3e3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Sep 2015 23:58:54 +0200 Subject: Revert change 0eb8c182131e: """Issue #23517: datetime.timedelta constructor now rounds microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding to nearest with ties going to nearest even integer (ROUND_HALF_EVEN).""" datetime.timedelta uses rounding mode ROUND_HALF_EVEN again. --- Lib/datetime.py | 4 ++-- Lib/test/datetimetester.py | 18 ++++++++++++------ Misc/NEWS | 7 +------ Modules/_datetimemodule.c | 22 +++++++++++++++++++++- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 6b2ac33..3c25ef8 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -407,7 +407,7 @@ class timedelta: # secondsfrac isn't referenced again if isinstance(microseconds, float): - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) seconds, microseconds = divmod(microseconds, 1000000) days, seconds = divmod(seconds, 24*3600) d += days @@ -418,7 +418,7 @@ class timedelta: days, seconds = divmod(seconds, 24*3600) d += days s += seconds - microseconds = _round_half_up(microseconds + usdouble) + microseconds = round(microseconds + usdouble) assert isinstance(s, int) assert isinstance(microseconds, int) assert abs(s) <= 3 * 24 * 3600 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f23add3..d87b106 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -663,14 +663,16 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): # Single-field rounding. eq(td(milliseconds=0.4/1000), td(0)) # rounds to 0 eq(td(milliseconds=-0.4/1000), td(0)) # rounds to 0 - eq(td(milliseconds=0.5/1000), td(microseconds=1)) - eq(td(milliseconds=-0.5/1000), td(microseconds=-1)) + eq(td(milliseconds=0.5/1000), td(microseconds=0)) + eq(td(milliseconds=-0.5/1000), td(microseconds=-0)) eq(td(milliseconds=0.6/1000), td(microseconds=1)) eq(td(milliseconds=-0.6/1000), td(microseconds=-1)) - eq(td(seconds=0.5/10**6), td(microseconds=1)) - eq(td(seconds=-0.5/10**6), td(microseconds=-1)) - eq(td(seconds=1/2**7), td(microseconds=7813)) - eq(td(seconds=-1/2**7), td(microseconds=-7813)) + eq(td(milliseconds=1.5/1000), td(microseconds=2)) + eq(td(milliseconds=-1.5/1000), td(microseconds=-2)) + eq(td(seconds=0.5/10**6), td(microseconds=0)) + eq(td(seconds=-0.5/10**6), td(microseconds=-0)) + eq(td(seconds=1/2**7), td(microseconds=7812)) + eq(td(seconds=-1/2**7), td(microseconds=-7812)) # Rounding due to contributions from more than one field. us_per_hour = 3600e6 @@ -683,6 +685,10 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): eq(td(hours=-.2/us_per_hour), td(0)) eq(td(days=-.4/us_per_day, hours=-.2/us_per_hour), td(microseconds=-1)) + # Test for a patch in Issue 8860 + eq(td(microseconds=0.5), 0.5*td(microseconds=1.0)) + eq(td(microseconds=0.5)//td.resolution, 0.5*td.resolution//td.resolution) + def test_massive_normalization(self): td = timedelta(microseconds=-1) self.assertEqual((td.days, td.seconds, td.microseconds), diff --git a/Misc/NEWS b/Misc/NEWS index c066a73..837288e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -17,18 +17,13 @@ Core and Builtins Library ------- -- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. +- Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of rounding towards -Infinity (ROUND_FLOOR). -- Issue #23517: datetime.timedelta constructor now rounds microseconds to - nearest with ties going away from zero (ROUND_HALF_UP), as Python 2 and - Python older than 3.3, instead of rounding to nearest with ties going to - nearest even integer (ROUND_HALF_EVEN). - - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ea2bae6..24e83d3 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -2149,9 +2149,29 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (leftover_us) { /* Round to nearest whole # of us, and add into x. */ double whole_us = round(leftover_us); + int x_is_odd; PyObject *temp; - whole_us = _PyTime_RoundHalfUp(leftover_us); + whole_us = round(leftover_us); + if (fabs(whole_us - leftover_us) == 0.5) { + /* We're exactly halfway between two integers. In order + * to do round-half-to-even, we must determine whether x + * is odd. Note that x is odd when it's last bit is 1. The + * code below uses bitwise and operation to check the last + * bit. */ + temp = PyNumber_And(x, one); /* temp <- x & 1 */ + if (temp == NULL) { + Py_DECREF(x); + goto Done; + } + x_is_odd = PyObject_IsTrue(temp); + Py_DECREF(temp); + if (x_is_odd == -1) { + Py_DECREF(x); + goto Done; + } + whole_us = 2.0 * round((leftover_us + x_is_odd) * 0.5) - x_is_odd; + } temp = PyLong_FromLong((long)whole_us); -- cgit v0.12 From 7667f58151d5efbbae4f0b1d7178f99dad0d74c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:02:23 +0200 Subject: Issue #23517: fromtimestamp() and utcfromtimestamp() methods of datetime.datetime now round microseconds to nearest with ties going to nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding towards -Infinity (ROUND_FLOOR). pytime API: replace _PyTime_ROUND_HALF_UP with _PyTime_ROUND_HALF_EVEN. Fix also _PyTime_Divide() for negative numbers. _PyTime_AsTimeval_impl() now reuses _PyTime_Divide() instead of reimplementing rounding modes. --- Include/pytime.h | 9 +- Lib/datetime.py | 2 +- Lib/test/datetimetester.py | 4 +- Lib/test/test_time.py | 245 ++++++++++++++++++++------------------------- Misc/NEWS | 6 +- Modules/_datetimemodule.c | 2 +- Modules/_testcapimodule.c | 2 +- Python/pytime.c | 71 ++++++------- 8 files changed, 152 insertions(+), 189 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 41fb806..54a0cc4 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -31,9 +31,9 @@ typedef enum { /* Round towards infinity (+inf). For example, used for timeout to wait "at least" N seconds. */ _PyTime_ROUND_CEILING=1, - /* Round to nearest with ties going away from zero. + /* Round to nearest with ties going to nearest even integer. For example, used to round from a Python float. */ - _PyTime_ROUND_HALF_UP + _PyTime_ROUND_HALF_EVEN } _PyTime_round_t; /* Convert a time_t to a PyLong. */ @@ -44,8 +44,9 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); -/* Round to nearest with ties going away from zero (_PyTime_ROUND_HALF_UP). */ -PyAPI_FUNC(double) _PyTime_RoundHalfUp( +/* Round to nearest with ties going to nearest even integer + (_PyTime_ROUND_HALF_EVEN) */ +PyAPI_FUNC(double) _PyTime_RoundHalfEven( double x); /* Convert a number of seconds, int or float, to time_t. */ diff --git a/Lib/datetime.py b/Lib/datetime.py index 3c25ef8..3f29bc4 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1380,7 +1380,7 @@ class datetime(date): A timezone info object may be passed in as well. """ frac, t = _math.modf(t) - us = _round_half_up(frac * 1e6) + us = round(frac * 1e6) if us >= 1000000: t += 1 us -= 1000000 diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d87b106..467fbe2 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1874,7 +1874,7 @@ class TestDateTime(TestDate): self.assertEqual(t, zero) t = fts(-1/2**7) self.assertEqual(t.second, 59) - self.assertEqual(t.microsecond, 992187) + self.assertEqual(t.microsecond, 992188) t = fts(1e-7) self.assertEqual(t, zero) @@ -1888,7 +1888,7 @@ class TestDateTime(TestDate): self.assertEqual(t.microsecond, 0) t = fts(1/2**7) self.assertEqual(t.second, 0) - self.assertEqual(t.microsecond, 7813) + self.assertEqual(t.microsecond, 7812) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d68dc4f..493b197 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -30,11 +30,11 @@ class _PyTime(enum.IntEnum): ROUND_FLOOR = 0 # Round towards infinity (+inf) ROUND_CEILING = 1 - # Round to nearest with ties going away from zero - ROUND_HALF_UP = 2 + # Round to nearest with ties going to nearest even integer + ROUND_HALF_EVEN = 2 ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_UP) + _PyTime.ROUND_HALF_EVEN) class TimeTestCase(unittest.TestCase): @@ -639,27 +639,26 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP - for obj, time_t, rnd in ( + HALF_EVEN = _PyTime.ROUND_HALF_EVEN + for obj, seconds, rnd in ( (-1.9, -2, FLOOR), (-1.9, -1, CEILING), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (1.9, 1, FLOOR), (1.9, 2, CEILING), - (1.9, 2, HALF_UP), - - # half up - (-0.999, -1, HALF_UP), - (-0.510, -1, HALF_UP), - (-0.500, -1, HALF_UP), - (-0.490, 0, HALF_UP), - ( 0.490, 0, HALF_UP), - ( 0.500, 1, HALF_UP), - ( 0.510, 1, HALF_UP), - ( 0.999, 1, HALF_UP), + (1.9, 2, HALF_EVEN), + + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) + with self.subTest(obj=obj, round=rnd, seconds=seconds): + self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) # Test OverflowError rnd = _PyTime.ROUND_FLOOR @@ -691,15 +690,15 @@ class TestPytime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( # Round towards minus infinity (-inf) (-1e-10, (0, 0), CEILING), (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), @@ -714,15 +713,13 @@ class TestPytime(unittest.TestCase): (-1.1234567890, (-2, 876543211), CEILING), (-1.1234567891, (-2, 876543211), CEILING), - # half up - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) @@ -823,10 +820,10 @@ class TestPyTime_t(unittest.TestCase): (-7.0, -7 * SEC_TO_NS), # nanosecond are kept for value <= 2^23 seconds, - # except 2**23-1e-9 with HALF_UP (2**22 - 1e-9, 4194303999999999), (2**22, 4194304000000000), (2**22 + 1e-9, 4194304000000001), + (2**23 - 1e-9, 8388607999999999), (2**23, 8388608000000000), # start loosing precision for value > 2^23 seconds @@ -859,38 +856,31 @@ class TestPyTime_t(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, ts, rnd in ( # close to zero ( 1e-10, 0, FLOOR), ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_UP), + ( 1e-10, 0, HALF_EVEN), (-1e-10, -1, FLOOR), (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_UP), + (-1e-10, 0, HALF_EVEN), # test rounding of the last nanosecond ( 1.1234567899, 1123456789, FLOOR), ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_UP), + ( 1.1234567899, 1123456790, HALF_EVEN), (-1.1234567899, -1123456790, FLOOR), (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_UP), + (-1.1234567899, -1123456790, HALF_EVEN), # close to 1 second ( 0.9999999999, 999999999, FLOOR), ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_UP), + ( 0.9999999999, 1000000000, HALF_EVEN), (-0.9999999999, -1000000000, FLOOR), (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_UP), - - # close to 2^23 seconds - (2**23 - 1e-9, 8388607999999999, FLOOR), - (2**23 - 1e-9, 8388607999999999, CEILING), - # Issue #23517: skip HALF_UP test because the result is different - # depending on the FPU and how the compiler optimize the code :-/ - #(2**23 - 1e-9, 8388608000000000, HALF_UP), + (-0.9999999999, -1000000000, HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timestamp=ts): self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) @@ -958,33 +948,23 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, tv, rnd in ( # nanoseconds (1, (0, 0), FLOOR), (1, (0, 1), CEILING), - (1, (0, 0), HALF_UP), + (1, (0, 0), HALF_EVEN), (-1, (-1, 999999), FLOOR), (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_UP), - - # seconds + nanoseconds - (1234567001, (1, 234567), FLOOR), - (1234567001, (1, 234568), CEILING), - (1234567001, (1, 234567), HALF_UP), - (-1234567001, (-2, 765432), FLOOR), - (-1234567001, (-2, 765433), CEILING), - (-1234567001, (-2, 765433), HALF_UP), - - # half up - (499, (0, 0), HALF_UP), - (500, (0, 1), HALF_UP), - (501, (0, 1), HALF_UP), - (999, (0, 1), HALF_UP), - (-499, (0, 0), HALF_UP), - (-500, (0, 0), HALF_UP), - (-501, (-1, 999999), HALF_UP), - (-999, (-1, 999999), HALF_UP), + (-1, (0, 0), HALF_EVEN), + + # half even + (-1500, (-1, 999998), HALF_EVEN), + (-999, (-1, 999999), HALF_EVEN), + (-500, (0, 0), HALF_EVEN), + (500, (0, 0), HALF_EVEN), + (999, (0, 1), HALF_EVEN), + (1500, (0, 2), HALF_EVEN), ): with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) @@ -1027,33 +1007,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * MS_TO_NS + 1, 1234, FLOOR), (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_UP), - (-1234 * MS_TO_NS - 1, -1234, FLOOR), - (-1234 * MS_TO_NS - 1, -1235, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_UP), + (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * MS_TO_NS - 1, -1235, FLOOR), + (-1234 * MS_TO_NS - 1, -1234, CEILING), + (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), # half up - (499999, 0, HALF_UP), - (499999, 0, HALF_UP), - (500000, 1, HALF_UP), - (999999, 1, HALF_UP), - (-499999, 0, HALF_UP), - (-500000, -1, HALF_UP), - (-500001, -1, HALF_UP), - (-999999, -1, HALF_UP), + (-1500000, -2, HALF_EVEN), + (-999999, -1, HALF_EVEN), + (-500000, 0, HALF_EVEN), + (500000, 0, HALF_EVEN), + (999999, 1, HALF_EVEN), + (1500000, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) @@ -1079,31 +1057,31 @@ class TestPyTime_t(unittest.TestCase): FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for ns, ms, rnd in ( # nanoseconds (1, 0, FLOOR), (1, 1, CEILING), - (1, 0, HALF_UP), - (-1, 0, FLOOR), - (-1, -1, CEILING), - (-1, 0, HALF_UP), + (1, 0, HALF_EVEN), + (-1, -1, FLOOR), + (-1, 0, CEILING), + (-1, 0, HALF_EVEN), # seconds + nanoseconds (1234 * US_TO_NS + 1, 1234, FLOOR), (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_UP), - (-1234 * US_TO_NS - 1, -1234, FLOOR), - (-1234 * US_TO_NS - 1, -1235, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_UP), + (1234 * US_TO_NS + 1, 1234, HALF_EVEN), + (-1234 * US_TO_NS - 1, -1235, FLOOR), + (-1234 * US_TO_NS - 1, -1234, CEILING), + (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), # half up - (1499, 1, HALF_UP), - (1500, 2, HALF_UP), - (1501, 2, HALF_UP), - (-1499, -1, HALF_UP), - (-1500, -2, HALF_UP), - (-1501, -2, HALF_UP), + (-1500, -2, HALF_EVEN), + (-999, -1, HALF_EVEN), + (-500, 0, HALF_EVEN), + (500, 0, HALF_EVEN), + (999, 1, HALF_EVEN), + (1500, 2, HALF_EVEN), ): with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) @@ -1142,23 +1120,23 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, time_t, rnd in ( (-1.9, -2, FLOOR), - (-1.9, -2, HALF_UP), + (-1.9, -2, HALF_EVEN), (-1.9, -1, CEILING), (1.9, 1, FLOOR), - (1.9, 2, HALF_UP), + (1.9, 2, HALF_EVEN), (1.9, 2, CEILING), - (-0.6, -1, HALF_UP), - (-0.5, -1, HALF_UP), - (-0.4, 0, HALF_UP), - - (0.4, 0, HALF_UP), - (0.5, 1, HALF_UP), - (0.6, 1, HALF_UP), + # half even + (-1.5, -2, HALF_EVEN), + (-0.9, -1, HALF_EVEN), + (-0.5, 0, HALF_EVEN), + ( 0.5, 0, HALF_EVEN), + ( 0.9, 1, HALF_EVEN), + ( 1.5, 2, HALF_EVEN), ): self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) @@ -1192,29 +1170,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timeval, rnd in ( (-1e-7, (-1, 999999), FLOOR), (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_UP), + (-1e-7, (0, 0), HALF_EVEN), (1e-7, (0, 0), FLOOR), (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_UP), + (1e-7, (0, 0), HALF_EVEN), (0.9999999, (0, 999999), FLOOR), (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_UP), - - (-0.6e-6, (-1, 999999), HALF_UP), - # skipped, -0.5e-6 is inexact in base 2 - #(-0.5e-6, (-1, 999999), HALF_UP), - (-0.4e-6, (0, 0), HALF_UP), - - (0.4e-6, (0, 0), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(0.5e-6, (0, 1), HALF_UP), - (0.6e-6, (0, 1), HALF_UP), + (0.9999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-6, (-1, 999998), HALF_EVEN), + (-0.9e-6, (-1, 999999), HALF_EVEN), + (-0.5e-6, (0, 0), HALF_EVEN), + (0.5e-6, (0, 0), HALF_EVEN), + (0.9e-6, (0, 1), HALF_EVEN), + (1.5e-6, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timeval=timeval): self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) @@ -1248,28 +1224,27 @@ class TestOldPyTime(unittest.TestCase): # Conversion giving different results depending on the rounding method FLOOR = _PyTime.ROUND_FLOOR CEILING = _PyTime.ROUND_CEILING - HALF_UP = _PyTime.ROUND_HALF_UP + HALF_EVEN = _PyTime.ROUND_HALF_EVEN for obj, timespec, rnd in ( (-1e-10, (-1, 999999999), FLOOR), (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_UP), + (-1e-10, (0, 0), HALF_EVEN), (1e-10, (0, 0), FLOOR), (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_UP), + (1e-10, (0, 0), HALF_EVEN), (0.9999999999, (0, 999999999), FLOOR), (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_UP), - - (-0.6e-9, (-1, 999999999), HALF_UP), - # skipped, 0.5e-6 is inexact in base 2 - #(-0.5e-9, (-1, 999999999), HALF_UP), - (-0.4e-9, (0, 0), HALF_UP), - - (0.4e-9, (0, 0), HALF_UP), - (0.5e-9, (0, 1), HALF_UP), - (0.6e-9, (0, 1), HALF_UP), + (0.9999999999, (1, 0), HALF_EVEN), + + # half even + (-1.5e-9, (-1, 999999998), HALF_EVEN), + (-0.9e-9, (-1, 999999999), HALF_EVEN), + (-0.5e-9, (0, 0), HALF_EVEN), + (0.5e-9, (0, 0), HALF_EVEN), + (0.9e-9, (0, 1), HALF_EVEN), + (1.5e-9, (0, 2), HALF_EVEN), ): with self.subTest(obj=obj, round=rnd, timespec=timespec): self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) diff --git a/Misc/NEWS b/Misc/NEWS index 837288e..c2f2e87 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -20,9 +20,9 @@ Library - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of - datetime.datetime now round microseconds to nearest with ties going away from - zero (ROUND_HALF_UP), as Python 2 and Python older than 3.3, instead of - rounding towards -Infinity (ROUND_FLOOR). + datetime.datetime now round microseconds to nearest with ties going to + nearest even integer (ROUND_HALF_EVEN), as round(float), instead of rounding + towards -Infinity (ROUND_FLOOR). - Issue #23552: Timeit now warns when there is substantial (4x) variance between best and worst times. Patch from Serhiy Storchaka. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 24e83d3..bbc51c6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4103,7 +4103,7 @@ datetime_from_timestamp(PyObject *cls, TM_FUNC f, PyObject *timestamp, long us; if (_PyTime_ObjectToTimeval(timestamp, - &timet, &us, _PyTime_ROUND_HALF_UP) == -1) + &timet, &us, _PyTime_ROUND_HALF_EVEN) == -1) return NULL; return datetime_from_timet_and_us(cls, f, timet, (int)us, tzinfo); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index c38d9ae..fed3286 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2648,7 +2648,7 @@ check_time_rounding(int round) { if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING - && round != _PyTime_ROUND_HALF_UP) { + && round != _PyTime_ROUND_HALF_EVEN) { PyErr_SetString(PyExc_ValueError, "invalid rounding"); return -1; } diff --git a/Python/pytime.c b/Python/pytime.c index 0364473..9f7ee2d 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -61,18 +61,15 @@ _PyLong_FromTime_t(time_t t) } double -_PyTime_RoundHalfUp(double x) +_PyTime_RoundHalfEven(double x) { - /* volatile avoids optimization changing how numbers are rounded */ - volatile double d = x; - if (d >= 0.0) - d = floor(d + 0.5); - else - d = ceil(d - 0.5); - return d; + double rounded = round(x); + if (fabs(x-rounded) == 0.5) + /* halfway case: round to even */ + rounded = 2.0*round(x/2.0); + return rounded; } - static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -84,8 +81,8 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); floatpart *= denominator; - if (round == _PyTime_ROUND_HALF_UP) - floatpart = _PyTime_RoundHalfUp(floatpart); + if (round == _PyTime_ROUND_HALF_EVEN) + floatpart = _PyTime_RoundHalfEven(floatpart); else if (round == _PyTime_ROUND_CEILING) floatpart = ceil(floatpart); else @@ -140,8 +137,8 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_HALF_UP) - d = _PyTime_RoundHalfUp(d); + if (round == _PyTime_ROUND_HALF_EVEN) + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else @@ -266,8 +263,8 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, d = value; d *= to_nanoseconds; - if (round == _PyTime_ROUND_HALF_UP) - d = _PyTime_RoundHalfUp(d); + if (round == _PyTime_ROUND_HALF_EVEN) + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) d = ceil(d); else @@ -351,14 +348,16 @@ _PyTime_AsNanosecondsObject(_PyTime_t t) } static _PyTime_t -_PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) +_PyTime_Divide(const _PyTime_t t, const _PyTime_t k, + const _PyTime_round_t round) { assert(k > 1); - if (round == _PyTime_ROUND_HALF_UP) { - _PyTime_t x, r; + if (round == _PyTime_ROUND_HALF_EVEN) { + _PyTime_t x, r, abs_r; x = t / k; r = t % k; - if (Py_ABS(r) >= k / 2) { + abs_r = Py_ABS(r); + if (abs_r > k / 2 || (abs_r == k / 2 && (Py_ABS(x) & 1))) { if (t >= 0) x++; else @@ -370,10 +369,14 @@ _PyTime_Divide(_PyTime_t t, _PyTime_t k, _PyTime_round_t round) if (t >= 0) return (t + k - 1) / k; else + return t / k; + } + else { + if (t >= 0) + return t / k; + else return (t - (k - 1)) / k; } - else - return t / k; } _PyTime_t @@ -392,17 +395,12 @@ static int _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - const long k = US_TO_NS; _PyTime_t secs, ns; int res = 0; int usec; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; - if (ns < 0) { - ns += SEC_TO_NS; - secs -= 1; - } #ifdef MS_WINDOWS /* On Windows, timeval.tv_sec is a long (32 bit), @@ -427,23 +425,12 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, res = -1; #endif - if (round == _PyTime_ROUND_HALF_UP) { - _PyTime_t r; - usec = (int)(ns / k); - r = ns % k; - if (Py_ABS(r) >= k / 2) { - if (ns >= 0) - usec++; - else - usec--; - } + usec = (int)_PyTime_Divide(ns, US_TO_NS, round); + if (usec < 0) { + usec += SEC_TO_US; + tv->tv_sec -= 1; } - else if (round == _PyTime_ROUND_CEILING) - usec = (int)((ns + k - 1) / k); - else - usec = (int)(ns / k); - - if (usec >= SEC_TO_US) { + else if (usec >= SEC_TO_US) { usec -= SEC_TO_US; tv->tv_sec += 1; } -- cgit v0.12 From cd5d765b0ee861509b14a1f21b630b8fb6eb258f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 01:09:21 +0200 Subject: cleanup datetime code remove scories of round half up code and debug code. --- Lib/datetime.py | 7 ------- Lib/test/datetimetester.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/datetime.py b/Lib/datetime.py index 3f29bc4..b734a74 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -316,13 +316,6 @@ def _divide_and_round(a, b): return q -def _round_half_up(x): - """Round to nearest with ties going away from zero.""" - if x >= 0.0: - return _math.floor(x + 0.5) - else: - return _math.ceil(x - 0.5) - class timedelta: """Represent the difference between two datetime objects. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 467fbe2..5d2df32 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -679,7 +679,7 @@ class TestTimeDelta(HarmlessMixedComparison, unittest.TestCase): us_per_day = us_per_hour * 24 eq(td(days=.4/us_per_day), td(0)) eq(td(hours=.2/us_per_hour), td(0)) - eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1), td) + eq(td(days=.4/us_per_day, hours=.2/us_per_hour), td(microseconds=1)) eq(td(days=-.4/us_per_day), td(0)) eq(td(hours=-.2/us_per_hour), td(0)) -- cgit v0.12 From df326eb1bb2210f42942a06fa386c1b45b8f2f0d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 9 Sep 2015 18:32:50 +0300 Subject: Fix versionchanged directive in datetime.rst --- Doc/library/datetime.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 9a0f936..5c659e7 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1756,8 +1756,8 @@ made to civil time. ``offset.minutes`` respectively. .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not - 'UTC+00:00'. + Name generated from ``offset=timedelta(0)`` is now plain 'UTC', not + 'UTC+00:00'. .. method:: timezone.dst(dt) -- cgit v0.12 From ce6aa749b40d39c8eb82463068cc58f974fbfdf5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:28:09 +0200 Subject: Make _PyTime_RoundHalfEven() private again --- Include/pytime.h | 5 ----- Python/pytime.c | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 54a0cc4..5de756a 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -44,11 +44,6 @@ PyAPI_FUNC(PyObject *) _PyLong_FromTime_t( PyAPI_FUNC(time_t) _PyLong_AsTime_t( PyObject *obj); -/* Round to nearest with ties going to nearest even integer - (_PyTime_ROUND_HALF_EVEN) */ -PyAPI_FUNC(double) _PyTime_RoundHalfEven( - double x); - /* Convert a number of seconds, int or float, to time_t. */ PyAPI_FUNC(int) _PyTime_ObjectToTime_t( PyObject *obj, diff --git a/Python/pytime.c b/Python/pytime.c index 9f7ee2d..37dfabb 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -60,7 +60,9 @@ _PyLong_FromTime_t(time_t t) #endif } -double +/* Round to nearest with ties going to nearest even integer + (_PyTime_ROUND_HALF_EVEN) */ +static double _PyTime_RoundHalfEven(double x) { double rounded = round(x); -- cgit v0.12 From 9ae47dfbd9b2f094205faad758b32803bea8e463 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:28:58 +0200 Subject: pytime: add _PyTime_Round() helper to factorize code --- Python/pytime.c | 45 ++++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 37dfabb..9735506 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -72,6 +72,17 @@ _PyTime_RoundHalfEven(double x) return rounded; } +static double +_PyTime_Round(double x, _PyTime_round_t round) +{ + if (round == _PyTime_ROUND_HALF_EVEN) + return _PyTime_RoundHalfEven(x); + else if (round == _PyTime_ROUND_CEILING) + return ceil(x); + else + return floor(x); +} + static int _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, double denominator, _PyTime_round_t round) @@ -83,12 +94,7 @@ _PyTime_DoubleToDenominator(double d, time_t *sec, long *numerator, floatpart = modf(d, &intpart); floatpart *= denominator; - if (round == _PyTime_ROUND_HALF_EVEN) - floatpart = _PyTime_RoundHalfEven(floatpart); - else if (round == _PyTime_ROUND_CEILING) - floatpart = ceil(floatpart); - else - floatpart = floor(floatpart); + floatpart = _PyTime_Round(floatpart, round); if (floatpart >= denominator) { floatpart -= denominator; intpart += 1.0; @@ -139,12 +145,7 @@ _PyTime_ObjectToTime_t(PyObject *obj, time_t *sec, _PyTime_round_t round) volatile double d; d = PyFloat_AsDouble(obj); - if (round == _PyTime_ROUND_HALF_EVEN) - d = _PyTime_RoundHalfEven(d); - else if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); + d = _PyTime_Round(d, round); (void)modf(d, &intpart); *sec = (time_t)intpart; @@ -255,7 +256,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) static int _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, - long to_nanoseconds) + long unit_to_ns) { double err; /* volatile avoids optimization changing how numbers are rounded */ @@ -263,14 +264,8 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, /* convert to a number of nanoseconds */ d = value; - d *= to_nanoseconds; - - if (round == _PyTime_ROUND_HALF_EVEN) - d = _PyTime_RoundHalfEven(d); - else if (round == _PyTime_ROUND_CEILING) - d = ceil(d); - else - d = floor(d); + d *= (double)unit_to_ns; + d = _PyTime_Round(d, round); *t = (_PyTime_t)d; err = d - (double)*t; @@ -283,12 +278,12 @@ _PyTime_FromFloatObject(_PyTime_t *t, double value, _PyTime_round_t round, static int _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, - long to_nanoseconds) + long unit_to_ns) { if (PyFloat_Check(obj)) { double d; d = PyFloat_AsDouble(obj); - return _PyTime_FromFloatObject(t, d, round, to_nanoseconds); + return _PyTime_FromFloatObject(t, d, round, unit_to_ns); } else { #ifdef HAVE_LONG_LONG @@ -305,8 +300,8 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, _PyTime_overflow(); return -1; } - *t = sec * to_nanoseconds; - if (*t / to_nanoseconds != sec) { + *t = sec * unit_to_ns; + if (*t / unit_to_ns != sec) { _PyTime_overflow(); return -1; } -- cgit v0.12 From 3e2c8d84c69063b8042a75ba04b5e75b1de0704e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 9 Sep 2015 22:32:48 +0200 Subject: test_time: rewrite PyTime API rounding tests Drop all hardcoded tests. Instead, reimplement each function in Python, usually using decimal.Decimal for the rounding mode. Add much more values to the dataset. Test various timestamp units from picroseconds to seconds, in integer and float. Enhance also _PyTime_AsSecondsDouble(). --- Lib/test/test_time.py | 796 +++++++++++++++----------------------------------- Python/pytime.c | 16 +- 2 files changed, 244 insertions(+), 568 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 493b197..bd697ae 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1,6 +1,8 @@ from test import support +import decimal import enum import locale +import math import platform import sys import sysconfig @@ -21,9 +23,11 @@ SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 TIME_MINYEAR = -TIME_MAXYEAR - 1 +SEC_TO_US = 10 ** 6 US_TO_NS = 10 ** 3 MS_TO_NS = 10 ** 6 SEC_TO_NS = 10 ** 9 +NS_TO_SEC = 10 ** 9 class _PyTime(enum.IntEnum): # Round towards minus infinity (-inf) @@ -33,8 +37,13 @@ class _PyTime(enum.IntEnum): # Round to nearest with ties going to nearest even integer ROUND_HALF_EVEN = 2 -ALL_ROUNDING_METHODS = (_PyTime.ROUND_FLOOR, _PyTime.ROUND_CEILING, - _PyTime.ROUND_HALF_EVEN) +# Rounding modes supported by PyTime +ROUNDING_MODES = ( + # (PyTime rounding method, decimal rounding method) + (_PyTime.ROUND_FLOOR, decimal.ROUND_FLOOR), + (_PyTime.ROUND_CEILING, decimal.ROUND_CEILING), + (_PyTime.ROUND_HALF_EVEN, decimal.ROUND_HALF_EVEN), +) class TimeTestCase(unittest.TestCase): @@ -610,126 +619,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - - @support.cpython_only - def test_time_t(self): - from _testcapi import pytime_object_to_time_t - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, seconds in ( - # int - (-1, -1), - (0, 0), - (1, 1), - - # float - (-1.0, -1), - (1.0, 1), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), - seconds) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, seconds, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -1, CEILING), - (-1.9, -2, HALF_EVEN), - - (1.9, 1, FLOOR), - (1.9, 2, CEILING), - (1.9, 2, HALF_EVEN), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, seconds=seconds): - self.assertEqual(pytime_object_to_time_t(obj, rnd), seconds) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) - - @support.cpython_only - def test_object_to_timespec(self): - from _testcapi import pytime_object_to_timespec - - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1/2**7, (-1, 992187500)), - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (1.0, (1, 0)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - # Round towards minus infinity (-inf) - (-1e-10, (0, 0), CEILING), - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), HALF_EVEN), - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - - (1.1234567890, (1, 123456789), FLOOR), - (1.1234567899, (1, 123456789), FLOOR), - (-1.1234567890, (-2, 876543210), FLOOR), - (-1.1234567891, (-2, 876543210), FLOOR), - # Round towards infinity (+inf) - (1.1234567890, (1, 123456790), CEILING), - (1.1234567899, (1, 123456790), CEILING), - (-1.1234567890, (-2, 876543211), CEILING), - (-1.1234567891, (-2, 876543211), CEILING), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): @@ -784,476 +673,259 @@ class TestPytime(unittest.TestCase): self.assertIs(lt.tm_zone, None) -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestPyTime_t(unittest.TestCase): +@unittest.skipIf(_testcapi is None, 'need the _testcapi module') +class CPyTimeTestCase: """ - Test the _PyTime_t API. + Base class to test the C _PyTime_t API. """ + OVERFLOW_SECONDS = None + + def _rounding_values(self, use_float): + "Build timestamps used to test rounding." + + units = [1, US_TO_NS, MS_TO_NS, SEC_TO_NS] + if use_float: + # picoseconds are only tested to pytime_converter accepting floats + units.append(1e-3) + + values = ( + # small values + 1, 2, 5, 7, 123, 456, 1234, + # 10^k - 1 + 9, + 99, + 999, + 9999, + 99999, + 999999, + # test half even rounding near 0.5, 1.5, 2.5, 3.5, 4.5 + 499, 500, 501, + 1499, 1500, 1501, + 2500, + 3500, + 4500, + ) + + ns_timestamps = [0] + for unit in units: + for value in values: + ns = value * unit + ns_timestamps.extend((-ns, ns)) + for pow2 in (0, 5, 10, 15, 22, 23, 24, 30, 33): + ns = (2 ** pow2) * SEC_TO_NS + ns_timestamps.extend(( + -ns-1, -ns, -ns+1, + ns-1, ns, ns+1 + )) + for seconds in (_testcapi.INT_MIN, _testcapi.INT_MAX): + ns_timestamps.append(seconds * SEC_TO_NS) + if use_float: + # numbers with an extract representation in IEEE 754 (base 2) + for pow2 in (3, 7, 10, 15): + ns = 2.0 ** (-pow2) + ns_timestamps.extend((-ns, ns)) + + # seconds close to _PyTime_t type limit + ns = (2 ** 63 // SEC_TO_NS) * SEC_TO_NS + ns_timestamps.extend((-ns, ns)) + + return ns_timestamps + + def _check_rounding(self, pytime_converter, expected_func, + use_float, unit_to_sec, value_filter=None): + + def convert_values(ns_timestamps): + if use_float: + unit_to_ns = SEC_TO_NS / float(unit_to_sec) + values = [ns / unit_to_ns for ns in ns_timestamps] + else: + unit_to_ns = SEC_TO_NS // unit_to_sec + values = [ns // unit_to_ns for ns in ns_timestamps] + + if value_filter: + values = filter(value_filter, values) + + # remove duplicates and sort + return sorted(set(values)) + + # test rounding + ns_timestamps = self._rounding_values(use_float) + valid_values = convert_values(ns_timestamps) + for time_rnd, decimal_rnd in ROUNDING_MODES : + context = decimal.getcontext() + context.rounding = decimal_rnd + + for value in valid_values: + expected = expected_func(value) + self.assertEqual(pytime_converter(value, time_rnd), + expected, + {'value': value, 'rounding': decimal_rnd}) + + # test overflow + ns = self.OVERFLOW_SECONDS * SEC_TO_NS + ns_timestamps = (-ns, ns) + overflow_values = convert_values(ns_timestamps) + for time_rnd, _ in ROUNDING_MODES : + for value in overflow_values: + with self.assertRaises(OverflowError): + pytime_converter(value, time_rnd) + + def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, + value_filter=None): + self._check_rounding(pytime_converter, expected_func, + False, unit_to_sec, value_filter) + + def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + self._check_rounding(pytime_converter, expected_func, + True, unit_to_sec) + + def decimal_round(self, x): + d = decimal.Decimal(x) + d = d.quantize(1) + return int(d) + + +class TestCPyTime(CPyTimeTestCase, unittest.TestCase): + """ + Test the C _PyTime_t API. + """ + # _PyTime_t is a 64-bit signed integer + OVERFLOW_SECONDS = math.ceil((2**63 + 1) / SEC_TO_NS) + def test_FromSeconds(self): from _testcapi import PyTime_FromSeconds - for seconds in (0, 3, -456, _testcapi.INT_MAX, _testcapi.INT_MIN): - with self.subTest(seconds=seconds): - self.assertEqual(PyTime_FromSeconds(seconds), - seconds * SEC_TO_NS) + + # PyTime_FromSeconds() expects a C int, reject values out of range + def c_int_filter(secs): + return (_testcapi.INT_MIN <= secs <= _testcapi.INT_MAX) + + self.check_int_rounding(lambda secs, rnd: PyTime_FromSeconds(secs), + lambda secs: secs * SEC_TO_NS, + value_filter=c_int_filter) def test_FromSecondsObject(self): from _testcapi import PyTime_FromSecondsObject - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, ts in ( - # integers - (0, 0), - (1, SEC_TO_NS), - (-3, -3 * SEC_TO_NS), - - # float: subseconds - (0.0, 0), - (1e-9, 1), - (1e-6, 10 ** 3), - (1e-3, 10 ** 6), - - # float: seconds - (2.0, 2 * SEC_TO_NS), - (123.0, 123 * SEC_TO_NS), - (-7.0, -7 * SEC_TO_NS), - - # nanosecond are kept for value <= 2^23 seconds, - (2**22 - 1e-9, 4194303999999999), - (2**22, 4194304000000000), - (2**22 + 1e-9, 4194304000000001), - (2**23 - 1e-9, 8388607999999999), - (2**23, 8388608000000000), - - # start loosing precision for value > 2^23 seconds - (2**23 + 1e-9, 8388608000000002), - - # nanoseconds are lost for value > 2^23 seconds - (2**24 - 1e-9, 16777215999999998), - (2**24, 16777216000000000), - (2**24 + 1e-9, 16777216000000000), - (2**25 - 1e-9, 33554432000000000), - (2**25 , 33554432000000000), - (2**25 + 1e-9, 33554432000000000), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036, 9223372036 * SEC_TO_NS), - (9223372036.0, 9223372036 * SEC_TO_NS), - (-9223372036, -9223372036 * SEC_TO_NS), - (-9223372036.0, -9223372036 * SEC_TO_NS), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) - - with self.subTest(round=rnd): - with self.assertRaises(OverflowError): - PyTime_FromSecondsObject(9223372037, rnd) - PyTime_FromSecondsObject(9223372037.0, rnd) - PyTime_FromSecondsObject(-9223372037, rnd) - PyTime_FromSecondsObject(-9223372037.0, rnd) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, ts, rnd in ( - # close to zero - ( 1e-10, 0, FLOOR), - ( 1e-10, 1, CEILING), - ( 1e-10, 0, HALF_EVEN), - (-1e-10, -1, FLOOR), - (-1e-10, 0, CEILING), - (-1e-10, 0, HALF_EVEN), - - # test rounding of the last nanosecond - ( 1.1234567899, 1123456789, FLOOR), - ( 1.1234567899, 1123456790, CEILING), - ( 1.1234567899, 1123456790, HALF_EVEN), - (-1.1234567899, -1123456790, FLOOR), - (-1.1234567899, -1123456789, CEILING), - (-1.1234567899, -1123456790, HALF_EVEN), - - # close to 1 second - ( 0.9999999999, 999999999, FLOOR), - ( 0.9999999999, 1000000000, CEILING), - ( 0.9999999999, 1000000000, HALF_EVEN), - (-0.9999999999, -1000000000, FLOOR), - (-0.9999999999, -999999999, CEILING), - (-0.9999999999, -1000000000, HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timestamp=ts): - self.assertEqual(PyTime_FromSecondsObject(obj, rnd), ts) + self.check_int_rounding( + PyTime_FromSecondsObject, + lambda secs: secs * SEC_TO_NS) + + self.check_float_rounding( + PyTime_FromSecondsObject, + lambda ns: self.decimal_round(ns * SEC_TO_NS)) def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble - for nanoseconds, seconds in ( - # near 1 nanosecond - ( 0, 0.0), - ( 1, 1e-9), - (-1, -1e-9), - - # near 1 second - (SEC_TO_NS + 1, 1.0 + 1e-9), - (SEC_TO_NS, 1.0), - (SEC_TO_NS - 1, 1.0 - 1e-9), - - # a few seconds - (123 * SEC_TO_NS, 123.0), - (-567 * SEC_TO_NS, -567.0), - - # nanosecond are kept for value <= 2^23 seconds - (4194303999999999, 2**22 - 1e-9), - (4194304000000000, 2**22), - (4194304000000001, 2**22 + 1e-9), - - # start loosing precision for value > 2^23 seconds - (8388608000000002, 2**23 + 1e-9), - - # nanoseconds are lost for value > 2^23 seconds - (16777215999999998, 2**24 - 1e-9), - (16777215999999999, 2**24 - 1e-9), - (16777216000000000, 2**24 ), - (16777216000000001, 2**24 ), - (16777216000000002, 2**24 + 2e-9), - - (33554432000000000, 2**25 ), - (33554432000000002, 2**25 ), - (33554432000000004, 2**25 + 4e-9), - - # close to 2^63 nanoseconds (_PyTime_t limit) - (9223372036 * SEC_TO_NS, 9223372036.0), - (-9223372036 * SEC_TO_NS, -9223372036.0), - ): - with self.subTest(nanoseconds=nanoseconds, seconds=seconds): - self.assertEqual(PyTime_AsSecondsDouble(nanoseconds), - seconds) - - def test_timeval(self): + def float_converter(ns): + if abs(ns) % SEC_TO_NS == 0: + return float(ns // SEC_TO_NS) + else: + return float(ns) / SEC_TO_NS + + self.check_int_rounding(lambda ns, rnd: PyTime_AsSecondsDouble(ns), + float_converter, + NS_TO_SEC) + + def create_decimal_converter(self, denominator): + denom = decimal.Decimal(denominator) + + def converter(value): + d = decimal.Decimal(value) / denom + return self.decimal_round(d) + + return converter + + def test_AsTimeval(self): from _testcapi import PyTime_AsTimeval - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (0, (0, 0)), - (1000, (0, 1)), - (-1000, (-1, 999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, tv, rnd in ( - # nanoseconds - (1, (0, 0), FLOOR), - (1, (0, 1), CEILING), - (1, (0, 0), HALF_EVEN), - (-1, (-1, 999999), FLOOR), - (-1, (0, 0), CEILING), - (-1, (0, 0), HALF_EVEN), - - # half even - (-1500, (-1, 999998), HALF_EVEN), - (-999, (-1, 999999), HALF_EVEN), - (-500, (0, 0), HALF_EVEN), - (500, (0, 0), HALF_EVEN), - (999, (0, 1), HALF_EVEN), - (1500, (0, 2), HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsTimeval(ns, rnd), tv) + + us_converter = self.create_decimal_converter(US_TO_NS) + + def timeval_converter(ns): + us = us_converter(ns) + return divmod(us, SEC_TO_US) + + self.check_int_rounding(PyTime_AsTimeval, + timeval_converter, + NS_TO_SEC) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') - def test_timespec(self): + def test_AsTimespec(self): from _testcapi import PyTime_AsTimespec - for ns, ts in ( - # nanoseconds - (0, (0, 0)), - (1, (0, 1)), - (-1, (-1, 999999999)), - - # seconds - (2 * SEC_TO_NS, (2, 0)), - (-3 * SEC_TO_NS, (-3, 0)), - - # seconds + nanoseconds - (1234567890, (1, 234567890)), - (-1234567890, (-2, 765432110)), - ): - with self.subTest(nanoseconds=ns, timespec=ts): - self.assertEqual(PyTime_AsTimespec(ns), ts) - - def test_milliseconds(self): + + def timespec_converter(ns): + return divmod(ns, SEC_TO_NS) + + self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), + timespec_converter, + NS_TO_SEC) + + def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # milliseconds - (1 * MS_TO_NS, 1), - (-2 * MS_TO_NS, -2), - - # seconds - (2 * SEC_TO_NS, 2000), - (-3 * SEC_TO_NS, -3000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * MS_TO_NS + 1, 1234, FLOOR), - (1234 * MS_TO_NS + 1, 1235, CEILING), - (1234 * MS_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * MS_TO_NS - 1, -1235, FLOOR), - (-1234 * MS_TO_NS - 1, -1234, CEILING), - (-1234 * MS_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500000, -2, HALF_EVEN), - (-999999, -1, HALF_EVEN), - (-500000, 0, HALF_EVEN), - (500000, 0, HALF_EVEN), - (999999, 1, HALF_EVEN), - (1500000, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMilliseconds(ns, rnd), ms) - - def test_microseconds(self): + + self.check_int_rounding(PyTime_AsMilliseconds, + self.create_decimal_converter(MS_TO_NS), + NS_TO_SEC) + + def test_AsMicroseconds(self): from _testcapi import PyTime_AsMicroseconds - for rnd in ALL_ROUNDING_METHODS: - for ns, tv in ( - # microseconds - (1 * US_TO_NS, 1), - (-2 * US_TO_NS, -2), - - # milliseconds - (1 * MS_TO_NS, 1000), - (-2 * MS_TO_NS, -2000), - - # seconds - (2 * SEC_TO_NS, 2000000), - (-3 * SEC_TO_NS, -3000000), - ): - with self.subTest(nanoseconds=ns, timeval=tv, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), tv) - - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for ns, ms, rnd in ( - # nanoseconds - (1, 0, FLOOR), - (1, 1, CEILING), - (1, 0, HALF_EVEN), - (-1, -1, FLOOR), - (-1, 0, CEILING), - (-1, 0, HALF_EVEN), - - # seconds + nanoseconds - (1234 * US_TO_NS + 1, 1234, FLOOR), - (1234 * US_TO_NS + 1, 1235, CEILING), - (1234 * US_TO_NS + 1, 1234, HALF_EVEN), - (-1234 * US_TO_NS - 1, -1235, FLOOR), - (-1234 * US_TO_NS - 1, -1234, CEILING), - (-1234 * US_TO_NS - 1, -1234, HALF_EVEN), - - # half up - (-1500, -2, HALF_EVEN), - (-999, -1, HALF_EVEN), - (-500, 0, HALF_EVEN), - (500, 0, HALF_EVEN), - (999, 1, HALF_EVEN), - (1500, 2, HALF_EVEN), - ): - with self.subTest(nanoseconds=ns, milliseconds=ms, round=rnd): - self.assertEqual(PyTime_AsMicroseconds(ns, rnd), ms) - - -@unittest.skipUnless(_testcapi is not None, - 'need the _testcapi module') -class TestOldPyTime(unittest.TestCase): + + self.check_int_rounding(PyTime_AsMicroseconds, + self.create_decimal_converter(US_TO_NS), + NS_TO_SEC) + + +class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): """ - Test the old _PyTime_t API: _PyTime_ObjectToXXX() functions. + Test the old C _PyTime_t API: _PyTime_ObjectToXXX() functions. """ - def setUp(self): - self.invalid_values = ( - -(2 ** 100), 2 ** 100, - -(2.0 ** 100.0), 2.0 ** 100.0, - ) - @support.cpython_only - def test_time_t(self): + # time_t is a 32-bit or 64-bit signed integer + OVERFLOW_SECONDS = 2 ** 64 + + def test_object_to_time_t(self): from _testcapi import pytime_object_to_time_t - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, time_t in ( - # int - (0, 0), - (-1, -1), - - # float - (1.0, 1), - (-1.0, -1), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, time_t, rnd in ( - (-1.9, -2, FLOOR), - (-1.9, -2, HALF_EVEN), - (-1.9, -1, CEILING), - - (1.9, 1, FLOOR), - (1.9, 2, HALF_EVEN), - (1.9, 2, CEILING), - - # half even - (-1.5, -2, HALF_EVEN), - (-0.9, -1, HALF_EVEN), - (-0.5, 0, HALF_EVEN), - ( 0.5, 0, HALF_EVEN), - ( 0.9, 1, HALF_EVEN), - ( 1.5, 2, HALF_EVEN), - ): - self.assertEqual(pytime_object_to_time_t(obj, rnd), time_t) - - # Test OverflowError - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_time_t, invalid, rnd) + self.check_int_rounding(pytime_object_to_time_t, + lambda secs: secs) + + self.check_float_rounding(pytime_object_to_time_t, + self.decimal_round) + + def create_converter(self, sec_to_unit): + def converter(secs): + floatpart, intpart = math.modf(secs) + intpart = int(intpart) + floatpart *= sec_to_unit + floatpart = self.decimal_round(floatpart) + if floatpart < 0: + floatpart += sec_to_unit + intpart -= 1 + elif floatpart >= sec_to_unit: + floatpart -= sec_to_unit + intpart += 1 + return (intpart, floatpart) + return converter def test_object_to_timeval(self): from _testcapi import pytime_object_to_timeval - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timeval in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (1/2**6, (0, 15625)), - (-1/2**6, (-1, 984375)), - (-1e-6, (-1, 999999)), - (1e-6, (0, 1)), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), - timeval) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timeval, rnd in ( - (-1e-7, (-1, 999999), FLOOR), - (-1e-7, (0, 0), CEILING), - (-1e-7, (0, 0), HALF_EVEN), - - (1e-7, (0, 0), FLOOR), - (1e-7, (0, 1), CEILING), - (1e-7, (0, 0), HALF_EVEN), - - (0.9999999, (0, 999999), FLOOR), - (0.9999999, (1, 0), CEILING), - (0.9999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-6, (-1, 999998), HALF_EVEN), - (-0.9e-6, (-1, 999999), HALF_EVEN), - (-0.5e-6, (0, 0), HALF_EVEN), - (0.5e-6, (0, 0), HALF_EVEN), - (0.9e-6, (0, 1), HALF_EVEN), - (1.5e-6, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timeval=timeval): - self.assertEqual(pytime_object_to_timeval(obj, rnd), timeval) - - rnd = _PyTime.ROUND_FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timeval, invalid, rnd) - - @support.cpython_only - def test_timespec(self): + self.check_int_rounding(pytime_object_to_timeval, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timeval, + self.create_converter(SEC_TO_US)) + + def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec - # Conversion giving the same result for all rounding methods - for rnd in ALL_ROUNDING_METHODS: - for obj, timespec in ( - # int - (0, (0, 0)), - (-1, (-1, 0)), - - # float - (-1.0, (-1, 0)), - (-1e-9, (-1, 999999999)), - (1e-9, (0, 1)), - (-1/2**9, (-1, 998046875)), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), - timespec) - - # Conversion giving different results depending on the rounding method - FLOOR = _PyTime.ROUND_FLOOR - CEILING = _PyTime.ROUND_CEILING - HALF_EVEN = _PyTime.ROUND_HALF_EVEN - for obj, timespec, rnd in ( - (-1e-10, (-1, 999999999), FLOOR), - (-1e-10, (0, 0), CEILING), - (-1e-10, (0, 0), HALF_EVEN), - - (1e-10, (0, 0), FLOOR), - (1e-10, (0, 1), CEILING), - (1e-10, (0, 0), HALF_EVEN), - - (0.9999999999, (0, 999999999), FLOOR), - (0.9999999999, (1, 0), CEILING), - (0.9999999999, (1, 0), HALF_EVEN), - - # half even - (-1.5e-9, (-1, 999999998), HALF_EVEN), - (-0.9e-9, (-1, 999999999), HALF_EVEN), - (-0.5e-9, (0, 0), HALF_EVEN), - (0.5e-9, (0, 0), HALF_EVEN), - (0.9e-9, (0, 1), HALF_EVEN), - (1.5e-9, (0, 2), HALF_EVEN), - ): - with self.subTest(obj=obj, round=rnd, timespec=timespec): - self.assertEqual(pytime_object_to_timespec(obj, rnd), timespec) - - # Test OverflowError - rnd = FLOOR - for invalid in self.invalid_values: - self.assertRaises(OverflowError, - pytime_object_to_timespec, invalid, rnd) + self.check_int_rounding(pytime_object_to_timespec, + lambda secs: (secs, 0)) + + self.check_float_rounding(pytime_object_to_timespec, + self.create_converter(SEC_TO_NS)) + if __name__ == "__main__": diff --git a/Python/pytime.c b/Python/pytime.c index 9735506..4c940c9 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -324,12 +324,16 @@ _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t roun double _PyTime_AsSecondsDouble(_PyTime_t t) { - _PyTime_t sec, ns; - /* Divide using integers to avoid rounding issues on the integer part. - 1e-9 cannot be stored exactly in IEEE 64-bit. */ - sec = t / SEC_TO_NS; - ns = t % SEC_TO_NS; - return (double)sec + (double)ns * 1e-9; + if (t % SEC_TO_NS == 0) { + _PyTime_t secs; + /* Divide using integers to avoid rounding issues on the integer part. + 1e-9 cannot be stored exactly in IEEE 64-bit. */ + secs = t / SEC_TO_NS; + return (double)secs; + } + else { + return (double)t / 1e9; + } } PyObject * -- cgit v0.12 From f5d72f35e854575772d971e35d64d643465019c6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 9 Sep 2015 22:39:44 -0400 Subject: Simply deque repeat by reusing code in in-line repeat. Avoid unnecessary division. --- Modules/_collectionsmodule.c | 48 +++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ea3e88f..db27e92 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -539,32 +539,6 @@ deque_concat(dequeobject *deque, PyObject *other) static void deque_clear(dequeobject *deque); static PyObject * -deque_repeat(dequeobject *deque, Py_ssize_t n) -{ - dequeobject *new_deque; - PyObject *result; - - /* XXX add a special case for when maxlen is defined */ - if (n < 0) - n = 0; - else if (n > 0 && Py_SIZE(deque) > MAX_DEQUE_LEN / n) - return PyErr_NoMemory(); - - new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); - new_deque->maxlen = deque->maxlen; - - for ( ; n ; n--) { - result = deque_extend(new_deque, (PyObject *)deque); - if (result == NULL) { - Py_DECREF(new_deque); - return NULL; - } - Py_DECREF(result); - } - return (PyObject *)new_deque; -} - -static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { Py_ssize_t i, size; @@ -583,10 +557,6 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } - if (size > MAX_DEQUE_LEN / n) { - return PyErr_NoMemory(); - } - if (size == 1) { /* common case, repeating a single element */ PyObject *item = deque->leftblock->data[deque->leftindex]; @@ -594,6 +564,9 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (deque->maxlen != -1 && n > deque->maxlen) n = deque->maxlen; + if (n > MAX_DEQUE_LEN) + return PyErr_NoMemory(); + for (i = 0 ; i < n-1 ; i++) { rv = deque_append(deque, item); if (rv == NULL) @@ -604,6 +577,10 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } + if ((size_t)size > MAX_DEQUE_LEN / (size_t)n) { + return PyErr_NoMemory(); + } + seq = PySequence_List((PyObject *)deque); if (seq == NULL) return seq; @@ -621,6 +598,17 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } +static PyObject * +deque_repeat(dequeobject *deque, Py_ssize_t n) +{ + dequeobject *new_deque; + + new_deque = (dequeobject *)deque_copy((PyObject *) deque); + if (new_deque == NULL) + return NULL; + return deque_inplace_repeat(new_deque, n); +} + /* The rotate() method is part of the public API and is used internally as a primitive for other methods. -- cgit v0.12 From 9c72f9b30a60d09c10b6cde23bd4b0546c6947af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 09:10:14 +0200 Subject: Fix test_time on Windows * Filter values which would overflow on conversion to the C long type (for timeval.tv_sec). * Adjust also the message of OverflowError on PyTime conversions * test_time: add debug information if a timestamp conversion fails --- Lib/test/test_time.py | 33 ++++++++++++++++++++++++--------- Python/pytime.c | 28 ++++++++-------------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index bd697ae..7f1613b 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -756,10 +756,15 @@ class CPyTimeTestCase: context.rounding = decimal_rnd for value in valid_values: - expected = expected_func(value) - self.assertEqual(pytime_converter(value, time_rnd), + debug_info = {'value': value, 'rounding': decimal_rnd} + try: + result = pytime_converter(value, time_rnd) + expected = expected_func(value) + except Exception as exc: + self.fail("Error on timestamp conversion: %s" % debug_info) + self.assertEqual(result, expected, - {'value': value, 'rounding': decimal_rnd}) + debug_info) # test overflow ns = self.OVERFLOW_SECONDS * SEC_TO_NS @@ -770,14 +775,15 @@ class CPyTimeTestCase: with self.assertRaises(OverflowError): pytime_converter(value, time_rnd) - def check_int_rounding(self, pytime_converter, expected_func, unit_to_sec=1, - value_filter=None): + def check_int_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, False, unit_to_sec, value_filter) - def check_float_rounding(self, pytime_converter, expected_func, unit_to_sec=1): + def check_float_rounding(self, pytime_converter, expected_func, + unit_to_sec=1, value_filter=None): self._check_rounding(pytime_converter, expected_func, - True, unit_to_sec) + True, unit_to_sec, value_filter) def decimal_round(self, x): d = decimal.Decimal(x) @@ -845,9 +851,19 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): us = us_converter(ns) return divmod(us, SEC_TO_US) + if sys.platform == 'win32': + from _testcapi import LONG_MIN, LONG_MAX + + # On Windows, timeval.tv_sec type is a C long + def seconds_filter(secs): + return LONG_MIN <= secs <= LONG_MAX + else: + seconds_filter = None + self.check_int_rounding(PyTime_AsTimeval, timeval_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=seconds_filter) @unittest.skipUnless(hasattr(_testcapi, 'PyTime_AsTimespec'), 'need _testcapi.PyTime_AsTimespec') @@ -927,6 +943,5 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): self.create_converter(SEC_TO_NS)) - if __name__ == "__main__": unittest.main() diff --git a/Python/pytime.c b/Python/pytime.c index 4c940c9..243f756 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -288,18 +288,21 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, else { #ifdef HAVE_LONG_LONG PY_LONG_LONG sec; - sec = PyLong_AsLongLong(obj); assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + + sec = PyLong_AsLongLong(obj); #else long sec; - sec = PyLong_AsLong(obj); assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + + sec = PyLong_AsLong(obj); #endif if (sec == -1 && PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_OverflowError)) _PyTime_overflow(); return -1; } + *t = sec * unit_to_ns; if (*t / unit_to_ns != sec) { _PyTime_overflow(); @@ -404,27 +407,12 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, ns = t % SEC_TO_NS; #ifdef MS_WINDOWS - /* On Windows, timeval.tv_sec is a long (32 bit), - whereas time_t can be 64-bit. */ - assert(sizeof(tv->tv_sec) == sizeof(long)); -#if SIZEOF_TIME_T > SIZEOF_LONG - if (secs > LONG_MAX) { - secs = LONG_MAX; - res = -1; - } - else if (secs < LONG_MIN) { - secs = LONG_MIN; - res = -1; - } -#endif tv->tv_sec = (long)secs; #else - /* On OpenBSD 5.4, timeval.tv_sec is a long. - Example: long is 64-bit, whereas time_t is 32-bit. */ tv->tv_sec = secs; +#endif if ((_PyTime_t)tv->tv_sec != secs) res = -1; -#endif usec = (int)_PyTime_Divide(ns, US_TO_NS, round); if (usec < 0) { @@ -440,7 +428,7 @@ _PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, tv->tv_usec = usec; if (res && raise) - _PyTime_overflow(); + error_time_t_overflow(); return res; } @@ -473,7 +461,7 @@ _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) ts->tv_nsec = nsec; if ((_PyTime_t)ts->tv_sec != secs) { - _PyTime_overflow(); + error_time_t_overflow(); return -1; } return 0; -- cgit v0.12 From 4237d3474c4792472009c6c6c5d46a7f7ab1410d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 10:10:39 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter values which would overflow when converted to a C time_t type. --- Lib/test/test_time.py | 23 ++++++++++++++++++----- Modules/_testcapimodule.c | 1 + 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 7f1613b..891c99d 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -680,6 +680,15 @@ class CPyTimeTestCase: """ OVERFLOW_SECONDS = None + def setUp(self): + from _testcapi import SIZEOF_TIME_T + bits = SIZEOF_TIME_T * 8 - 1 + self.time_t_min = -2 ** bits + self.time_t_max = 2 ** bits - 1 + + def time_t_filter(self, seconds): + return (self.time_t_min <= seconds <= self.time_t_max) + def _rounding_values(self, use_float): "Build timestamps used to test rounding." @@ -858,7 +867,7 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): def seconds_filter(secs): return LONG_MIN <= secs <= LONG_MAX else: - seconds_filter = None + seconds_filter = self.time_t_filter self.check_int_rounding(PyTime_AsTimeval, timeval_converter, @@ -875,7 +884,8 @@ class TestCPyTime(CPyTimeTestCase, unittest.TestCase): self.check_int_rounding(lambda ns, rnd: PyTime_AsTimespec(ns), timespec_converter, - NS_TO_SEC) + NS_TO_SEC, + value_filter=self.time_t_filter) def test_AsMilliseconds(self): from _testcapi import PyTime_AsMilliseconds @@ -904,7 +914,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_time_t self.check_int_rounding(pytime_object_to_time_t, - lambda secs: secs) + lambda secs: secs, + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, self.decimal_round) @@ -928,7 +939,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timeval self.check_int_rounding(pytime_object_to_timeval, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, self.create_converter(SEC_TO_US)) @@ -937,7 +949,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): from _testcapi import pytime_object_to_timespec self.check_int_rounding(pytime_object_to_timespec, - lambda secs: (secs, 0)) + lambda secs: (secs, 0), + value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, self.create_converter(SEC_TO_NS)) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index fed3286..1f013c2 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4114,6 +4114,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "PY_SSIZE_T_MAX", PyLong_FromSsize_t(PY_SSIZE_T_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MIN", PyLong_FromSsize_t(PY_SSIZE_T_MIN)); PyModule_AddObject(m, "SIZEOF_PYGC_HEAD", PyLong_FromSsize_t(sizeof(PyGC_Head))); + PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); -- cgit v0.12 From 350b51839aea3f5bd541b960ef1ec1b5a170854a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 11:45:06 +0200 Subject: Fix test_time on platform with 32-bit time_t type Filter also values for check_float_rounding(). --- Lib/test/test_time.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 891c99d..30b01d5 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -918,7 +918,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_time_t, - self.decimal_round) + self.decimal_round, + value_filter=self.time_t_filter) def create_converter(self, sec_to_unit): def converter(secs): @@ -943,7 +944,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timeval, - self.create_converter(SEC_TO_US)) + self.create_converter(SEC_TO_US), + value_filter=self.time_t_filter) def test_object_to_timespec(self): from _testcapi import pytime_object_to_timespec @@ -953,7 +955,8 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): value_filter=self.time_t_filter) self.check_float_rounding(pytime_object_to_timespec, - self.create_converter(SEC_TO_NS)) + self.create_converter(SEC_TO_NS), + value_filter=self.time_t_filter) if __name__ == "__main__": -- cgit v0.12 From 1efbebaac2a29ae42f3de8d74071ea2eda660711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 11:48:00 +0200 Subject: Try to fix test_time.test_AsSecondsDouble() on "x86 Gentoo Non-Debug with X 3.x" buildbot Use volatile keyword in _PyTime_Round() --- Python/pytime.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 243f756..9470636 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -75,12 +75,17 @@ _PyTime_RoundHalfEven(double x) static double _PyTime_Round(double x, _PyTime_round_t round) { + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d; + + d = x; if (round == _PyTime_ROUND_HALF_EVEN) - return _PyTime_RoundHalfEven(x); + d = _PyTime_RoundHalfEven(d); else if (round == _PyTime_ROUND_CEILING) - return ceil(x); + d = ceil(d); else - return floor(x); + d = floor(d); + return d; } static int -- cgit v0.12 From ff0ed3e71cb828103cf21442231686f1b348479b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 13:25:17 +0200 Subject: New try to fix test_time.test_AsSecondsDouble() on x86 buildbots. Use volatile keyword in _PyTime_AsSecondsDouble() --- Python/pytime.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9470636..c82d598 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -332,16 +332,21 @@ _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t roun double _PyTime_AsSecondsDouble(_PyTime_t t) { + /* volatile avoids optimization changing how numbers are rounded */ + volatile double d; + if (t % SEC_TO_NS == 0) { _PyTime_t secs; /* Divide using integers to avoid rounding issues on the integer part. 1e-9 cannot be stored exactly in IEEE 64-bit. */ secs = t / SEC_TO_NS; - return (double)secs; + d = (double)secs; } else { - return (double)t / 1e9; + d = (double)t; + d /= 1e9; } + return d; } PyObject * -- cgit v0.12 From c60542b12bdf11487b959bbb304f4ea194be6a19 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 15:55:07 +0200 Subject: pytime: add _PyTime_check_mul_overflow() macro to avoid undefined behaviour Overflow test in test_FromSecondsObject() fails on FreeBSD 10.0 buildbot which uses clang. clang implements more aggressive optimization which gives different result than GCC on undefined behaviours. Check if a multiplication will overflow, instead of checking if a multiplicatin had overflowed, to avoid undefined behaviour. Add also debug information if the test on overflow fails. --- Lib/test/test_time.py | 3 ++- Python/pytime.c | 35 ++++++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 30b01d5..f883c45 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -781,7 +781,8 @@ class CPyTimeTestCase: overflow_values = convert_values(ns_timestamps) for time_rnd, _ in ROUNDING_MODES : for value in overflow_values: - with self.assertRaises(OverflowError): + debug_info = {'value': value, 'rounding': time_rnd} + with self.assertRaises(OverflowError, msg=debug_info): pytime_converter(value, time_rnd) def check_int_rounding(self, pytime_converter, expected_func, diff --git a/Python/pytime.c b/Python/pytime.c index c82d598..1b20dc7 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -7,6 +7,11 @@ #include /* mach_absolute_time(), mach_timebase_info() */ #endif +#define _PyTime_check_mul_overflow(a, b) \ + (assert(b > 0), \ + (_PyTime_t)(a) < _PyTime_MIN / (_PyTime_t)(b) \ + || _PyTime_MAX / (_PyTime_t)(b) < (_PyTime_t)(a)) + /* To millisecond (10^-3) */ #define SEC_TO_MS 1000 @@ -226,12 +231,15 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) _PyTime_t t; int res = 0; - t = (_PyTime_t)ts->tv_sec * SEC_TO_NS; - if (t / SEC_TO_NS != ts->tv_sec) { + assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + t = (_PyTime_t)ts->tv_sec; + + if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { if (raise) _PyTime_overflow(); res = -1; } + t = t * SEC_TO_NS; t += ts->tv_nsec; @@ -245,12 +253,15 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - t = (_PyTime_t)tv->tv_sec * SEC_TO_NS; - if (t / SEC_TO_NS != tv->tv_sec) { + assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + t = (_PyTime_t)tv->tv_sec; + + if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { if (raise) _PyTime_overflow(); res = -1; } + t = t * SEC_TO_NS; t += (_PyTime_t)tv->tv_usec * US_TO_NS; @@ -308,11 +319,11 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, return -1; } - *t = sec * unit_to_ns; - if (*t / unit_to_ns != sec) { + if (_PyTime_check_mul_overflow(sec, unit_to_ns)) { _PyTime_overflow(); return -1; } + *t = sec * unit_to_ns; return 0; } } @@ -587,19 +598,20 @@ _PyTime_GetSystemClockWithInfo(_PyTime_t *t, _Py_clock_info_t *info) return pygettimeofday_new(t, info, 1); } - static int pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) { #if defined(MS_WINDOWS) - ULONGLONG result; + ULONGLONG ticks; + _PyTime_t t; assert(info == NULL || raise); - result = GetTickCount64(); + ticks = GetTickCount64(); + assert(sizeof(result) <= sizeof(_PyTime_t)); + t = (_PyTime_t)ticks; - *tp = result * MS_TO_NS; - if (*tp / MS_TO_NS != result) { + if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { if (raise) { _PyTime_overflow(); return -1; @@ -607,6 +619,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) /* Hello, time traveler! */ assert(0); } + *tp = t * MS_TO_NS; if (info) { DWORD timeAdjustment, timeIncrement; -- cgit v0.12 From 51b93984449ec79908036cc7a1a5922458f87690 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 10 Sep 2015 16:00:06 +0200 Subject: pytime: oops, fix typos on Windows --- Python/pytime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 1b20dc7..9d0b0fa 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -253,7 +253,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)tv->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -608,7 +608,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) assert(info == NULL || raise); ticks = GetTickCount64(); - assert(sizeof(result) <= sizeof(_PyTime_t)); + assert(sizeof(ticks) <= sizeof(_PyTime_t)); t = (_PyTime_t)ticks; if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { -- cgit v0.12 From 67c78b5421ce9ebba1399e5935879ca01d5ac091 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 12 Sep 2015 11:00:20 -0400 Subject: In-line the append operations inside deque_inplace_repeat(). --- Modules/_collectionsmodule.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index db27e92..7f81460 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -567,12 +567,26 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (n > MAX_DEQUE_LEN) return PyErr_NoMemory(); + deque->state++; for (i = 0 ; i < n-1 ; i++) { - rv = deque_append(deque, item); - if (rv == NULL) - return NULL; - Py_DECREF(rv); + if (deque->rightindex == BLOCKLEN - 1) { + block *b = newblock(Py_SIZE(deque) + i); + if (b == NULL) { + Py_SIZE(deque) += i; + return NULL; + } + b->leftlink = deque->rightblock; + CHECK_END(deque->rightblock->rightlink); + deque->rightblock->rightlink = b; + deque->rightblock = b; + MARK_END(b->rightlink); + deque->rightindex = -1; + } + deque->rightindex++; + Py_INCREF(item); + deque->rightblock->data[deque->rightindex] = item; } + Py_SIZE(deque) += i; Py_INCREF(deque); return (PyObject *)deque; } -- cgit v0.12 From 6408dc82fabf5bbbc8b7ed7287e34b4e0080191d Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 12 Sep 2015 18:53:36 -0400 Subject: Fixed indentation. --- Parser/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 04baeaf..5046fa5 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1741,7 +1741,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) else { end_quote_size = 0; if (c == '\\') - c = tok_nextc(tok); /* skip escaped char */ + c = tok_nextc(tok); /* skip escaped char */ } } -- cgit v0.12 From 95e2cc5d122c8469c3fd3040f43311547a561663 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Sep 2015 02:41:18 -0400 Subject: Fix refcount. --- Modules/_collectionsmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7f81460..17233e4 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -616,11 +616,14 @@ static PyObject * deque_repeat(dequeobject *deque, Py_ssize_t n) { dequeobject *new_deque; + PyObject *rv; new_deque = (dequeobject *)deque_copy((PyObject *) deque); if (new_deque == NULL) return NULL; - return deque_inplace_repeat(new_deque, n); + rv = deque_inplace_repeat(new_deque, n); + Py_DECREF(new_deque); + return rv; } /* The rotate() method is part of the public API and is used internally -- cgit v0.12 From e4f3467df17873db60dd1e8daf61e0c629abb130 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Sep 2015 19:27:01 -0400 Subject: Add an exact type match fast path for deque_copy(). --- Modules/_collectionsmodule.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 17233e4..49a46a1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1205,6 +1205,22 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_copy(PyObject *deque) { + if (Py_TYPE(deque) == &deque_type) { + dequeobject *new_deque; + PyObject *rv; + + new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); + if (new_deque == NULL) + return NULL; + new_deque->maxlen = ((dequeobject *)deque)->maxlen; + rv = deque_extend(new_deque, deque); + if (rv != NULL) { + Py_DECREF(rv); + return (PyObject *)new_deque; + } + Py_DECREF(new_deque); + return NULL; + } if (((dequeobject *)deque)->maxlen == -1) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else -- cgit v0.12 From ad26225e1a62f32848669b09326835dfd0369a79 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 14 Sep 2015 01:03:04 -0400 Subject: Tighten inner-loop for deque_inplace_repeat(). --- Modules/_collectionsmodule.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 49a46a1..087f8e5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -568,7 +568,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return PyErr_NoMemory(); deque->state++; - for (i = 0 ; i < n-1 ; i++) { + for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque) + i); if (b == NULL) { @@ -582,9 +582,11 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) MARK_END(b->rightlink); deque->rightindex = -1; } - deque->rightindex++; - Py_INCREF(item); - deque->rightblock->data[deque->rightindex] = item; + for ( ; i < n-1 && deque->rightindex != BLOCKLEN - 1 ; i++) { + deque->rightindex++; + Py_INCREF(item); + deque->rightblock->data[deque->rightindex] = item; + } } Py_SIZE(deque) += i; Py_INCREF(deque); -- cgit v0.12 From f11d0d2c0d44c812534067c76e61c366b5b79a06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 12:15:59 +0200 Subject: Issue #25122: try to debug test_eintr hang on FreeBSD * Add verbose mode to test_eintr * Always enable verbose mode in test_eintr * Use faulthandler.dump_traceback_later() with a timeout of 15 minutes in eintr_tester.py --- Lib/test/eintrdata/eintr_tester.py | 8 ++++++++ Lib/test/test_eintr.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 0616df6..b96f09b 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import faulthandler import io import os import select @@ -36,12 +37,19 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) + if hasattr(faulthandler, 'dump_traceback_later'): + # Most tests take less than 30 seconds, so 15 minutes should be + # enough. dump_traceback_later() is implemented with a thread, but + # pthread_sigmask() is used to mask all signaled on this thread. + faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 111ead3..1d1d16e 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -1,5 +1,7 @@ import os import signal +import subprocess +import sys import unittest from test import support @@ -14,7 +16,15 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + + # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD + if True: #support.verbose: + args = [sys.executable, tester] + with subprocess.Popen(args, stdout=sys.stderr) as proc: + exitcode = proc.wait() + self.assertEqual(exitcode, 0) + else: + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v0.12 From 44879a0b18048e2d722a866e3ec0485012bce732 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:38:09 +0200 Subject: Issue #25122: Fix test_eintr, kill child process on error Some test_eintr hangs on waiting for the child process completion if an error occurred on the parent. Kill the child process on error (in the parent) to avoid the hang. --- Lib/test/eintrdata/eintr_tester.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index b96f09b..e1330a7 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -8,6 +8,7 @@ Signals are generated in-process using setitimer(ITIMER_REAL), which allows sub-second periodicity (contrarily to signal()). """ +import contextlib import faulthandler import io import os @@ -21,6 +22,16 @@ import unittest from test import support +@contextlib.contextmanager +def kill_on_error(proc): + """Context manager killing the subprocess if a Python exception is raised.""" + with proc: + try: + yield proc + except: + proc.kill() + raise + @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") class EINTRBaseTest(unittest.TestCase): @@ -38,7 +49,7 @@ class EINTRBaseTest(unittest.TestCase): def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 15 minutes should be + # Most tests take less than 30 seconds, so 5 minutes should be # enough. dump_traceback_later() is implemented with a thread, but # pthread_sigmask() is used to mask all signaled on this thread. faulthandler.dump_traceback_later(5 * 60, exit=True) @@ -120,7 +131,8 @@ class OSEINTRTest(EINTRBaseTest): ' os.write(wr, data)', )) - with self.subprocess(code, str(wr), pass_fds=[wr]) as proc: + proc = self.subprocess(code, str(wr), pass_fds=[wr]) + with kill_on_error(proc): os.close(wr) for data in datas: self.assertEqual(data, os.read(rd, len(data))) @@ -156,7 +168,8 @@ class OSEINTRTest(EINTRBaseTest): ' % (len(value), data_len))', )) - with self.subprocess(code, str(rd), pass_fds=[rd]) as proc: + proc = self.subprocess(code, str(rd), pass_fds=[rd]) + with kill_on_error(proc): os.close(rd) written = 0 while written < len(data): @@ -198,7 +211,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = wr.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): wr.close() for data in datas: self.assertEqual(data, recv_func(rd, len(data))) @@ -248,7 +261,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = rd.fileno() proc = self.subprocess(code, str(fd), pass_fds=[fd]) - with proc: + with kill_on_error(proc): rd.close() written = 0 while written < len(data): @@ -288,7 +301,8 @@ class SocketEINTRTest(EINTRBaseTest): ' time.sleep(sleep_time)', )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): client_sock, _ = sock.accept() client_sock.close() self.assertEqual(proc.wait(), 0) @@ -315,7 +329,8 @@ class SocketEINTRTest(EINTRBaseTest): do_open_close_reader, )) - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): do_open_close_writer(filename) self.assertEqual(proc.wait(), 0) @@ -372,7 +387,8 @@ class SignalEINTRTest(EINTRBaseTest): )) t0 = time.monotonic() - with self.subprocess(code) as proc: + proc = self.subprocess(code) + with kill_on_error(proc): # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v0.12 From 3731bbe8b10f7f31240fbffe7d9617365f7814dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 22:55:52 +0200 Subject: Issue #25122: test_eintr: don't redirect stdout to stderr sys.stderr is sometimes a StringIO. The redirection was just a hack to see eintr_tester.py output in red in the buildbot output. --- Lib/test/test_eintr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index 1d1d16e..a1907e8 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -20,7 +20,7 @@ class EINTRTests(unittest.TestCase): # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD if True: #support.verbose: args = [sys.executable, tester] - with subprocess.Popen(args, stdout=sys.stderr) as proc: + with subprocess.Popen(args) as proc: exitcode = proc.wait() self.assertEqual(exitcode, 0) else: -- cgit v0.12 From 1e0f8ecdd8a4fb1fa21f4da1a8849ffd42b966f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Sep 2015 23:59:00 +0200 Subject: Issue #25122: optimize test_eintr Fix test_write(): copy support.PIPE_MAX_SIZE bytes, not support.PIPE_MAX_SIZE*3 bytes. --- Lib/test/eintrdata/eintr_tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1330a7..24e809a 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -144,14 +144,14 @@ class OSEINTRTest(EINTRBaseTest): # rd closed explicitly by parent # we must write enough data for the write() to block - data = b"xyz" * support.PIPE_MAX_SIZE + data = b"x" * support.PIPE_MAX_SIZE code = '\n'.join(( 'import io, os, sys, time', '', 'rd = int(sys.argv[1])', 'sleep_time = %r' % self.sleep_time, - 'data = b"xyz" * %s' % support.PIPE_MAX_SIZE, + 'data = b"x" * %s' % support.PIPE_MAX_SIZE, 'data_len = len(data)', '', '# let the parent block on write()', -- cgit v0.12 From e41b4712fe884551aef7dd3b8852907dcc310a29 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Sep 2015 09:23:28 +0200 Subject: Issue #25122: add debug traces to test_eintr.test_open() --- Lib/test/eintrdata/eintr_tester.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 24e809a..c3c08fe 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -326,12 +326,26 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', + 'print("try to open %a fifo for reading, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', + '', do_open_close_reader, + '', + 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' + ' % (path, os.getpid(), os.getppid()),' + ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): + print("try to open %a fifo for writing, pid %s" + % (filename, os.getpid()), + flush=True) do_open_close_writer(filename) + print("%a fifo opened for writing and closed, pid %s" + % (filename, os.getpid()), + flush=True) self.assertEqual(proc.wait(), 0) -- cgit v0.12 From 6db1fd5fb8b2287cd4713c95f0df451e375c8853 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 21:49:12 -0700 Subject: Close issue24840: Enum._value_ is queried for bool(); original patch by Mike Lundy --- Lib/enum.py | 3 +++ Lib/test/test_enum.py | 7 +++++++ Misc/ACKS | 1 + 3 files changed, 11 insertions(+) diff --git a/Lib/enum.py b/Lib/enum.py index c28f345..616b2ea 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -476,6 +476,9 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) + def __bool__(self): + return bool(self._value_) + def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 4b5d0d0..0f7b769 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -270,6 +270,13 @@ class TestEnum(unittest.TestCase): class Wrong(Enum): _any_name_ = 9 + def test_bool(self): + class Logic(Enum): + true = True + false = False + self.assertTrue(Logic.true) + self.assertFalse(Logic.false) + def test_contains(self): Season = self.Season self.assertIn(Season.AUTUMN, Season) diff --git a/Misc/ACKS b/Misc/ACKS index e26be93..43b4c99 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -877,6 +877,7 @@ Kang-Hao (Kenny) Lu Lukas Lueg Loren Luke Fredrik Lundh +Mike Lundy Zhongyue Luo Mark Lutz Taras Lyapun -- cgit v0.12 From e5754ab0c4ab44cd1e7e5484ce07e640964387dd Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:03:52 -0700 Subject: Close issue25147: use C implementation of OrderedDict --- Lib/enum.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/enum.py b/Lib/enum.py index 616b2ea..6284b9b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,7 +1,12 @@ import sys -from collections import OrderedDict from types import MappingProxyType, DynamicClassAttribute +try: + from _collections import OrderedDict +except ImportError: + from collections import OrderedDict + + __all__ = ['Enum', 'IntEnum', 'unique'] -- cgit v0.12 From c791507e1fd1ce218f40c7bd4b85076c278ae8d9 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Thu, 17 Sep 2015 22:55:40 -0700 Subject: Issue 25147: add reason for using _collections --- Lib/enum.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/enum.py b/Lib/enum.py index 6284b9b..8d04e2d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,6 +1,7 @@ import sys from types import MappingProxyType, DynamicClassAttribute +# try _collections first to reduce startup cost try: from _collections import OrderedDict except ImportError: -- cgit v0.12 From baab5f73417dabd5e5dc6fe9e0dc3b917f812fb2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 11:23:42 +0200 Subject: Issue #25122: Fix test_eintr.test_open() on FreeBSD Skip test_open() and test_os_open(): both tests uses a FIFO and signals, but there is a bug in the FreeBSD kernel which blocks the test. Skip the tests until the bug is fixed in FreeBSD kernel. Remove also debug traces from test_eintr: * stop using faulthandler to have a timeout * remove print() Write also open and close on two lines in test_open() and test_os_open() tests. If these tests block again, we can know if the test is stuck at open or close. test_eintr: don't always run the test in debug mode. --- Lib/test/eintrdata/eintr_tester.py | 44 +++++++++++++++----------------------- Lib/test/test_eintr.py | 3 +-- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index c3c08fe..c2407ca 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,7 +9,6 @@ sub-second periodicity (contrarily to signal()). """ import contextlib -import faulthandler import io import os import select @@ -48,19 +47,12 @@ class EINTRBaseTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.orig_handler = signal.signal(signal.SIGALRM, lambda *args: None) - if hasattr(faulthandler, 'dump_traceback_later'): - # Most tests take less than 30 seconds, so 5 minutes should be - # enough. dump_traceback_later() is implemented with a thread, but - # pthread_sigmask() is used to mask all signaled on this thread. - faulthandler.dump_traceback_later(5 * 60, exit=True) signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) - if hasattr(faulthandler, 'cancel_dump_traceback_later'): - faulthandler.cancel_dump_traceback_later() @classmethod def tearDownClass(cls): @@ -307,6 +299,11 @@ class SocketEINTRTest(EINTRBaseTest): client_sock.close() self.assertEqual(proc.wait(), 0) + # Issue #25122: There is a race condition in the FreeBSD kernel on + # handling signals in the FIFO device. Skip the test until the bug is + # fixed in the kernel. Bet that the bug will be fixed in FreeBSD 11. + # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 + @support.requires_freebsd_version(11) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN @@ -326,36 +323,29 @@ class SocketEINTRTest(EINTRBaseTest): '# let the parent block', 'time.sleep(sleep_time)', '', - 'print("try to open %a fifo for reading, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', - '', do_open_close_reader, - '', - 'print("%a fifo opened for reading and closed, pid %s, ppid %s"' - ' % (path, os.getpid(), os.getppid()),' - ' flush=True)', )) proc = self.subprocess(code) with kill_on_error(proc): - print("try to open %a fifo for writing, pid %s" - % (filename, os.getpid()), - flush=True) do_open_close_writer(filename) - print("%a fifo opened for writing and closed, pid %s" - % (filename, os.getpid()), - flush=True) - self.assertEqual(proc.wait(), 0) + def python_open(self, path): + fp = open(path, 'w') + fp.close() + def test_open(self): - self._test_open("open(path, 'r').close()", - lambda path: open(path, 'w').close()) + self._test_open("fp = open(path, 'r')\nfp.close()", + self.python_open) + + def os_open(self, path): + fd = os.open(path, os.O_WRONLY) + os.close(fd) def test_os_open(self): - self._test_open("os.close(os.open(path, os.O_RDONLY))", - lambda path: os.close(os.open(path, os.O_WRONLY))) + self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", + self.os_open) @unittest.skipUnless(hasattr(signal, "setitimer"), "requires setitimer()") diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index a1907e8..aabad83 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -17,8 +17,7 @@ class EINTRTests(unittest.TestCase): # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - # FIXME: Issue #25122, always run in verbose mode to debug hang on FreeBSD - if True: #support.verbose: + if support.verbose: args = [sys.executable, tester] with subprocess.Popen(args) as proc: exitcode = proc.wait() -- cgit v0.12 From 1e2b6882fc28a3cded227f55e4dc3f937afd678e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 13:23:02 +0200 Subject: Issue #25155: Add _PyTime_AsTimevalTime_t() function On Windows, the tv_sec field of the timeval structure has the type C long, whereas it has the type C time_t on all other platforms. A C long has a size of 32 bits (signed inter, 1 bit for the sign, 31 bits for the value) which is not enough to store an Epoch timestamp after the year 2038. Add the _PyTime_AsTimevalTime_t() function written for datetime.datetime.now(): convert a _PyTime_t timestamp to a (secs, us) tuple where secs type is time_t. It allows to support dates after the year 2038 on Windows. Enhance also _PyTime_AsTimeval_impl() to detect overflow on the number of seconds when rounding the number of microseconds. --- Include/pytime.h | 12 +++++++ Modules/_datetimemodule.c | 9 +++--- Python/pytime.c | 79 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 77 insertions(+), 23 deletions(-) diff --git a/Include/pytime.h b/Include/pytime.h index 5de756a..98612e1 100644 --- a/Include/pytime.h +++ b/Include/pytime.h @@ -120,6 +120,18 @@ PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); +/* Convert a timestamp to a number of seconds (secs) and microseconds (us). + us is always positive. This function is similar to _PyTime_AsTimeval() + except that secs is always a time_t type, whereas the timeval structure + uses a C long for tv_sec on Windows. + Raise an exception and return -1 if the conversion overflowed, + return 0 on success. */ +PyAPI_FUNC(int) _PyTime_AsTimevalTime_t( + _PyTime_t t, + time_t *secs, + int *us, + _PyTime_round_t round); + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) /* Convert a timestamp to a timespec structure (nanosecond resolution). tv_nsec is always positive. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index bbc51c6..5289222 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4117,13 +4117,14 @@ static PyObject * datetime_best_possible(PyObject *cls, TM_FUNC f, PyObject *tzinfo) { _PyTime_t ts = _PyTime_GetSystemClock(); - struct timeval tv; + time_t secs; + int us; - if (_PyTime_AsTimeval(ts, &tv, _PyTime_ROUND_FLOOR) < 0) + if (_PyTime_AsTimevalTime_t(ts, &secs, &us, _PyTime_ROUND_FLOOR) < 0) return NULL; - assert(0 <= tv.tv_usec && tv.tv_usec <= 999999); + assert(0 <= us && us <= 999999); - return datetime_from_timet_and_us(cls, f, tv.tv_sec, tv.tv_usec, tzinfo); + return datetime_from_timet_and_us(cls, f, secs, us, tzinfo); } /*[clinic input] diff --git a/Python/pytime.c b/Python/pytime.c index 9d0b0fa..9889a3b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -417,54 +417,95 @@ _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round) } static int -_PyTime_AsTimeval_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, - int raise) +_PyTime_AsTimeval_impl(_PyTime_t t, _PyTime_t *p_secs, int *p_us, + _PyTime_round_t round) { _PyTime_t secs, ns; - int res = 0; int usec; + int res = 0; secs = t / SEC_TO_NS; ns = t % SEC_TO_NS; -#ifdef MS_WINDOWS - tv->tv_sec = (long)secs; -#else - tv->tv_sec = secs; -#endif - if ((_PyTime_t)tv->tv_sec != secs) - res = -1; - usec = (int)_PyTime_Divide(ns, US_TO_NS, round); if (usec < 0) { usec += SEC_TO_US; - tv->tv_sec -= 1; + if (secs != _PyTime_MIN) + secs -= 1; + else + res = -1; } else if (usec >= SEC_TO_US) { usec -= SEC_TO_US; - tv->tv_sec += 1; + if (secs != _PyTime_MAX) + secs += 1; + else + res = -1; } - assert(0 <= usec && usec < SEC_TO_US); - tv->tv_usec = usec; - if (res && raise) - error_time_t_overflow(); + *p_secs = secs; + *p_us = usec; + return res; } +static int +_PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, + _PyTime_round_t round, int raise) +{ + _PyTime_t secs; + int us; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, &us, round); + +#ifdef MS_WINDOWS + tv->tv_sec = (long)secs; +#else + tv->tv_sec = secs; +#endif + tv->tv_usec = us; + + if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + if (raise) + error_time_t_overflow(); + return -1; + } + return 0; +} + int _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) { - return _PyTime_AsTimeval_impl(t, tv, round, 1); + return _PyTime_AsTimevalStruct_impl(t, tv, round, 1); } int _PyTime_AsTimeval_noraise(_PyTime_t t, struct timeval *tv, _PyTime_round_t round) { - return _PyTime_AsTimeval_impl(t, tv, round, 0); + return _PyTime_AsTimevalStruct_impl(t, tv, round, 0); } +int +_PyTime_AsTimevalTime_t(_PyTime_t t, time_t *p_secs, int *us, + _PyTime_round_t round) +{ + _PyTime_t secs; + int res; + + res = _PyTime_AsTimeval_impl(t, &secs, us, round); + + *p_secs = secs; + + if (res < 0 || (_PyTime_t)*p_secs != secs) { + error_time_t_overflow(); + return -1; + } + return 0; +} + + #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_KQUEUE) int _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts) -- cgit v0.12 From 5ebfe42cdf7ba4ba0a573e2990960fea460540b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 14:52:15 +0200 Subject: Oops, fix test_microsecond_rounding() Test self.theclass, not datetime. Regression introduced by manual tests. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5d2df32..a6c7421 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1851,8 +1851,8 @@ class TestDateTime(TestDate): 18000 + 3600 + 2*60 + 3 + 4*1e-6) def test_microsecond_rounding(self): - for fts in (datetime.fromtimestamp, - self.theclass.utcfromtimestamp): + for fts in [self.theclass.fromtimestamp, + self.theclass.utcfromtimestamp]: zero = fts(0) self.assertEqual(zero.second, 0) self.assertEqual(zero.microsecond, 0) -- cgit v0.12 From 3abf44e48f973d5ffba779471a6c4dab00be2f77 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 15:38:37 +0200 Subject: Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function instead of the getentropy() function. The getentropy() function is blocking to generate very good quality entropy, os.urandom() doesn't need such high-quality entropy. --- Misc/NEWS | 5 +++++ Python/random.c | 49 ++++++++++++++++++++++++++++++++++--------------- configure | 43 ++++++++++++++++++++++++++++++++++++++++--- configure.ac | 31 ++++++++++++++++++++++++++++--- pyconfig.h.in | 3 +++ 5 files changed, 110 insertions(+), 21 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 827d788..77efd05 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,11 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the + getrandom() function instead of the getentropy() function. The getentropy() + function is blocking to generate very good quality entropy, os.urandom() + doesn't need such high-quality entropy. + - Issue #9232: Modify Python's grammar to allow trailing commas in the argument list of a function declaration. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. diff --git a/Python/random.c b/Python/random.c index ea09e84..8f3e6d6 100644 --- a/Python/random.c +++ b/Python/random.c @@ -6,7 +6,9 @@ # ifdef HAVE_SYS_STAT_H # include # endif -# ifdef HAVE_GETRANDOM_SYSCALL +# ifdef HAVE_GETRANDOM +# include +# elif defined(HAVE_GETRANDOM_SYSCALL) # include # endif #endif @@ -70,7 +72,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise) return 0; } -#elif HAVE_GETENTROPY +#elif defined(HAVE_GETENTROPY) && !defined(sun) +#define PY_GETENTROPY + /* Fill buffer with size pseudo-random bytes generated by getentropy(). Return 0 on success, or raise an exception and return -1 on error. @@ -105,16 +109,19 @@ py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal) return 0; } -#else /* !HAVE_GETENTROPY */ +#else + +#if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL) +#define PY_GETRANDOM -#ifdef HAVE_GETRANDOM_SYSCALL static int py_getrandom(void *buffer, Py_ssize_t size, int raise) { - /* is getrandom() supported by the running kernel? - * need Linux kernel 3.17 or later */ + /* Is getrandom() supported by the running kernel? + * Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */ static int getrandom_works = 1; - /* Use /dev/urandom, block if the kernel has no entropy */ + /* Use non-blocking /dev/urandom device. On Linux at boot, the getrandom() + * syscall blocks until /dev/urandom is initialized with enough entropy. */ const int flags = 0; int n; @@ -124,7 +131,18 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) while (0 < size) { errno = 0; - /* Use syscall() because the libc doesn't expose getrandom() yet, see: +#ifdef HAVE_GETRANDOM + if (raise) { + Py_BEGIN_ALLOW_THREADS + n = getrandom(buffer, size, flags); + Py_END_ALLOW_THREADS + } + else { + n = getrandom(buffer, size, flags); + } +#else + /* On Linux, use the syscall() function because the GNU libc doesn't + * expose the Linux getrandom() syscall yet. See: * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */ if (raise) { Py_BEGIN_ALLOW_THREADS @@ -134,6 +152,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise) else { n = syscall(SYS_getrandom, buffer, size, flags); } +#endif if (n < 0) { if (errno == ENOSYS) { @@ -182,7 +201,7 @@ dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size) assert (0 < size); -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM if (py_getrandom(buffer, size, 0) == 1) return; /* getrandom() is not supported by the running kernel, fall back @@ -218,14 +237,14 @@ dev_urandom_python(char *buffer, Py_ssize_t size) int fd; Py_ssize_t n; struct _Py_stat_struct st; -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM int res; #endif if (size <= 0) return 0; -#ifdef HAVE_GETRANDOM_SYSCALL +#ifdef PY_GETRANDOM res = py_getrandom(buffer, size, 1); if (res < 0) return -1; @@ -304,7 +323,7 @@ dev_urandom_close(void) } } -#endif /* HAVE_GETENTROPY */ +#endif /* Fill buffer with pseudo-random bytes generated by a linear congruent generator (LCG): @@ -345,7 +364,7 @@ _PyOS_URandom(void *buffer, Py_ssize_t size) #ifdef MS_WINDOWS return win32_urandom((unsigned char *)buffer, size, 1); -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY return py_getentropy(buffer, size, 0); #else return dev_urandom_python((char*)buffer, size); @@ -392,7 +411,7 @@ _PyRandom_Init(void) else { #ifdef MS_WINDOWS (void)win32_urandom(secret, secret_size, 0); -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY (void)py_getentropy(secret, secret_size, 1); #else dev_urandom_noraise(secret, secret_size); @@ -408,7 +427,7 @@ _PyRandom_Fini(void) CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } -#elif HAVE_GETENTROPY +#elif PY_GETENTROPY /* nothing to clean */ #else dev_urandom_close(); diff --git a/configure b/configure index 95d953a..f7b1839 100755 --- a/configure +++ b/configure @@ -16005,11 +16005,11 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include int main() { - const int flags = 0; char buffer[1]; - int n; + const size_t buflen = sizeof(buffer); + const int flags = 0; /* ignore the result, Python checks for ENOSYS at runtime */ - (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags); + (void)syscall(SYS_getrandom, buffer, buflen, flags); return 0; } @@ -16031,6 +16031,43 @@ $as_echo "#define HAVE_GETRANDOM_SYSCALL 1" >>confdefs.h fi +# check if the getrandom() function is available +# the test was written for the Solaris function of +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for the getrandom() function" >&5 +$as_echo_n "checking for the getrandom() function... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + + int main() { + char buffer[1]; + const size_t buflen = sizeof(buffer); + const int flags = 0; + /* ignore the result, Python checks for ENOSYS at runtime */ + (void)getrandom(buffer, buflen, flags); + return 0; + } + + +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + have_getrandom=yes +else + have_getrandom=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_getrandom" >&5 +$as_echo "$have_getrandom" >&6; } + +if test "$have_getrandom" = yes; then + +$as_echo "#define HAVE_GETRANDOM 1" >>confdefs.h + +fi + # generate output files ac_config_files="$ac_config_files Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh" diff --git a/configure.ac b/configure.ac index 668234d..180a421 100644 --- a/configure.ac +++ b/configure.ac @@ -5111,11 +5111,11 @@ AC_LINK_IFELSE( #include int main() { - const int flags = 0; char buffer[1]; - int n; + const size_t buflen = sizeof(buffer); + const int flags = 0; /* ignore the result, Python checks for ENOSYS at runtime */ - (void)syscall(SYS_getrandom, buffer, sizeof(buffer), flags); + (void)syscall(SYS_getrandom, buffer, buflen, flags); return 0; } ]]) @@ -5127,6 +5127,31 @@ if test "$have_getrandom_syscall" = yes; then [Define to 1 if the Linux getrandom() syscall is available]) fi +# check if the getrandom() function is available +# the test was written for the Solaris function of +AC_MSG_CHECKING(for the getrandom() function) +AC_LINK_IFELSE( +[ + AC_LANG_SOURCE([[ + #include + + int main() { + char buffer[1]; + const size_t buflen = sizeof(buffer); + const int flags = 0; + /* ignore the result, Python checks for ENOSYS at runtime */ + (void)getrandom(buffer, buflen, flags); + return 0; + } + ]]) +],[have_getrandom=yes],[have_getrandom=no]) +AC_MSG_RESULT($have_getrandom) + +if test "$have_getrandom" = yes; then + AC_DEFINE(HAVE_GETRANDOM, 1, + [Define to 1 if the getrandom() function is available]) +fi + # generate output files AC_CONFIG_FILES(Makefile.pre Modules/Setup.config Misc/python.pc Misc/python-config.sh) AC_CONFIG_FILES([Modules/ld_so_aix], [chmod +x Modules/ld_so_aix]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 0d40c94..b0cafb3 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -395,6 +395,9 @@ /* Define to 1 if you have the `getpwent' function. */ #undef HAVE_GETPWENT +/* Define to 1 if the getrandom() function is available */ +#undef HAVE_GETRANDOM + /* Define to 1 if the Linux getrandom() syscall is available */ #undef HAVE_GETRANDOM_SYSCALL -- cgit v0.12 From d8f432a98c6bbe143002f45e181b7e5243d1166e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2015 16:24:31 +0200 Subject: Issue #25003: Skip test_os.URandomFDTests on Solaris 11.3 and newer When os.urandom() is implemented with the getrandom() function, it doesn't use a file descriptor. --- Lib/test/test_os.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b73b64b..d4b9e9c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1225,13 +1225,15 @@ class URandomTests(unittest.TestCase): self.assertNotEqual(data1, data2) -HAVE_GETENTROPY = (sysconfig.get_config_var('HAVE_GETENTROPY') == 1) -HAVE_GETRANDOM = (sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) - -@unittest.skipIf(HAVE_GETENTROPY, - "getentropy() does not use a file descriptor") -@unittest.skipIf(HAVE_GETRANDOM, - "getrandom() does not use a file descriptor") +# os.urandom() doesn't use a file descriptor when it is implemented with the +# getentropy() function, the getrandom() function or the getrandom() syscall +OS_URANDOM_DONT_USE_FD = ( + sysconfig.get_config_var('HAVE_GETENTROPY') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) + +@unittest.skipIf(OS_URANDOM_DONT_USE_FD , + "os.random() does not use a file descriptor") class URandomFDTests(unittest.TestCase): @unittest.skipUnless(resource, "test requires the resource module") def test_urandom_failure(self): -- cgit v0.12 From 0e14e6610b3cf692377996fad569496191a3f731 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 19 Sep 2015 00:21:33 -0600 Subject: Hoist constant expression out of an inner loop --- Modules/_collectionsmodule.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 087f8e5..dbe44e1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -371,6 +371,7 @@ static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -417,7 +418,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - deque_trim_left(deque); + if (trim) + deque_trim_left(deque); } if (PyErr_Occurred()) { Py_DECREF(it); @@ -434,6 +436,7 @@ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -480,7 +483,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - deque_trim_right(deque); + if (trim) + deque_trim_right(deque); } if (PyErr_Occurred()) { Py_DECREF(it); -- cgit v0.12 From aed8830af3bb5a79878cf0f603ebbd8a37f5b36e Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 19 Sep 2015 09:05:42 -0700 Subject: Add a fast path (no iterator creation) for a common case for repeating deques of size 1 --- Lib/test/test_deque.py | 9 +++++++++ Modules/_collectionsmodule.c | 15 +++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 8718716..c61e80b 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -654,6 +654,15 @@ class TestBasic(unittest.TestCase): self.assertNotEqual(id(d), id(e)) self.assertEqual(list(d), list(e)) + for i in range(5): + for maxlen in range(-1, 6): + s = [random.random() for j in range(i)] + d = deque(s) if maxlen == -1 else deque(s, maxlen) + e = d.copy() + self.assertEqual(d, e) + self.assertEqual(d.maxlen, e.maxlen) + self.assertTrue(all(x is y for x, y in zip(d, e))) + def test_copy_method(self): mut = [10] d = deque([mut]) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index dbe44e1..3e2c69f 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1211,6 +1211,7 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_copy(PyObject *deque) { + dequeobject *old_deque = (dequeobject *)deque; if (Py_TYPE(deque) == &deque_type) { dequeobject *new_deque; PyObject *rv; @@ -1218,8 +1219,14 @@ deque_copy(PyObject *deque) new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); if (new_deque == NULL) return NULL; - new_deque->maxlen = ((dequeobject *)deque)->maxlen; - rv = deque_extend(new_deque, deque); + new_deque->maxlen = old_deque->maxlen; + /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; + rv = deque_append(new_deque, item); + } else { + rv = deque_extend(new_deque, deque); + } if (rv != NULL) { Py_DECREF(rv); return (PyObject *)new_deque; @@ -1227,11 +1234,11 @@ deque_copy(PyObject *deque) Py_DECREF(new_deque); return NULL; } - if (((dequeobject *)deque)->maxlen == -1) + if (old_deque->maxlen == -1) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", - deque, ((dequeobject *)deque)->maxlen, NULL); + deque, old_deque->maxlen, NULL); } PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); -- cgit v0.12 From 235a6f09847ad554d8bf073d4e1d58d1e398ae8c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 14:51:32 -0400 Subject: Issue #24965: Implement PEP 498 "Literal String Interpolation". Documentation is still needed, I'll open an issue for that. --- Include/Python-ast.h | 23 +- Lib/test/test_fstring.py | 715 ++++++++++++++++++++++++++++++++++ Misc/NEWS | 5 + Parser/Python.asdl | 2 + Parser/tokenizer.c | 8 +- Python/Python-ast.c | 165 ++++++++ Python/ast.c | 985 ++++++++++++++++++++++++++++++++++++++++++++--- Python/compile.c | 117 +++++- Python/symtable.c | 8 + 9 files changed, 1965 insertions(+), 63 deletions(-) create mode 100644 Lib/test/test_fstring.py diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 3bc015f..ea6679c 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -201,9 +201,10 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, Await_kind=12, Yield_kind=13, YieldFrom_kind=14, Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, - Bytes_kind=19, NameConstant_kind=20, Ellipsis_kind=21, - Attribute_kind=22, Subscript_kind=23, Starred_kind=24, - Name_kind=25, List_kind=26, Tuple_kind=27}; + FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, + NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, + Subscript_kind=25, Starred_kind=26, Name_kind=27, + List_kind=28, Tuple_kind=29}; struct _expr { enum _expr_kind kind; union { @@ -297,6 +298,16 @@ struct _expr { } Str; struct { + expr_ty value; + int conversion; + expr_ty format_spec; + } FormattedValue; + + struct { + asdl_seq *values; + } JoinedStr; + + struct { bytes s; } Bytes; @@ -543,6 +554,12 @@ expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena); #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3) expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena); +#define FormattedValue(a0, a1, a2, a3, a4, a5) _Py_FormattedValue(a0, a1, a2, a3, a4, a5) +expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, + int lineno, int col_offset, PyArena *arena); +#define JoinedStr(a0, a1, a2, a3) _Py_JoinedStr(a0, a1, a2, a3) +expr_ty _Py_JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena + *arena); #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3) expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena); #define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py new file mode 100644 index 0000000..a6ff9cf --- /dev/null +++ b/Lib/test/test_fstring.py @@ -0,0 +1,715 @@ +import ast +import types +import decimal +import unittest + +a_global = 'global variable' + +# You could argue that I'm too strict in looking for specific error +# values with assertRaisesRegex, but without it it's way too easy to +# make a syntax error in the test strings. Especially with all of the +# triple quotes, raw strings, backslashes, etc. I think it's a +# worthwhile tradeoff. When I switched to this method, I found many +# examples where I wasn't testing what I thought I was. + +class TestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test__format__lookup(self): + # Make sure __format__ is looked up on the type, not the instance. + class X: + def __format__(self, spec): + return 'class' + + x = X() + + # Add a bound __format__ method to the 'y' instance, but not + # the 'x' instance. + y = X() + y.__format__ = types.MethodType(lambda self, spec: 'instance', y) + + self.assertEqual(f'{y}', format(y)) + self.assertEqual(f'{y}', 'class') + self.assertEqual(format(x), format(y)) + + # __format__ is not called this way, but still make sure it + # returns what we expect (so we can make sure we're bypassing + # it). + self.assertEqual(x.__format__(''), 'class') + self.assertEqual(y.__format__(''), 'instance') + + # This is how __format__ is actually called. + self.assertEqual(type(x).__format__(x, ''), 'class') + self.assertEqual(type(y).__format__(y, ''), 'class') + + def test_ast(self): + # Inspired by http://bugs.python.org/issue24975 + class X: + def __init__(self): + self.called = False + def __call__(self): + self.called = True + return 4 + x = X() + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + + # Make sure x was not called. + self.assertFalse(x.called) + + # Actually run the code. + exec(c) + + # Make sure x was called. + self.assertTrue(x.called) + + def test_literal_eval(self): + # With no expressions, an f-string is okay. + self.assertEqual(ast.literal_eval("f'x'"), 'x') + self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy') + + # But this should raise an error. + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'x{3}'") + + # As should this, which uses a different ast node + with self.assertRaisesRegex(ValueError, 'malformed node or string'): + ast.literal_eval("f'{3}'") + + def test_ast_compile_time_concat(self): + x = [''] + + expr = """x[0] = 'foo' f'{3}'""" + t = ast.parse(expr) + c = compile(t, '', 'exec') + exec(c) + self.assertEqual(x[0], 'foo3') + + def test_literal(self): + self.assertEqual(f'', '') + self.assertEqual(f'a', 'a') + self.assertEqual(f' ', ' ') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\N{GREEK CAPITAL LETTER DELTA}') + self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', + '\u0394') + self.assertEqual(f'\N{True}', '\u22a8') + self.assertEqual(rf'\N{True}', r'\NTrue') + + def test_escape_order(self): + # note that hex(ord('{')) == 0x7b, so this + # string becomes f'a{4*10}b' + self.assertEqual(f'a\u007b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10}b', 'a40b') + self.assertEqual(f'a\x7b4*10\N{RIGHT CURLY BRACKET}b', 'a40b') + self.assertEqual(f'{"a"!\N{LATIN SMALL LETTER R}}', "'a'") + self.assertEqual(f'{10\x3a02X}', '0A') + self.assertEqual(f'{10:02\N{LATIN CAPITAL LETTER X}}', '0A') + + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + [r"""f'a{\u007b4*10}b'""", # mis-matched brackets + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation character', + [r"""f'{"a"\!r}'""", + r"""f'{a\!r}'""", + ]) + + def test_unterminated_string(self): + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r"""f'{"x'""", + r"""f'{"x}'""", + r"""f'{("x'""", + r"""f'{("x}'""", + ]) + + def test_mismatched_parens(self): + self.assertAllRaise(SyntaxError, 'f-string: mismatched', + ["f'{((}'", + ]) + + def test_double_braces(self): + self.assertEqual(f'{{', '{') + self.assertEqual(f'a{{', 'a{') + self.assertEqual(f'{{b', '{b') + self.assertEqual(f'a{{b', 'a{b') + self.assertEqual(f'}}', '}') + self.assertEqual(f'a}}', 'a}') + self.assertEqual(f'}}b', '}b') + self.assertEqual(f'a}}b', 'a}b') + + self.assertEqual(f'{{{10}', '{10') + self.assertEqual(f'}}{10}', '}10') + self.assertEqual(f'}}{{{10}', '}{10') + self.assertEqual(f'}}a{{{10}', '}a{10') + + self.assertEqual(f'{10}{{', '10{') + self.assertEqual(f'{10}}}', '10}') + self.assertEqual(f'{10}}}{{', '10}{') + self.assertEqual(f'{10}}}a{{' '}', '10}a{}') + + # Inside of strings, don't interpret doubled brackets. + self.assertEqual(f'{"{{}}"}', '{{}}') + + self.assertAllRaise(TypeError, 'unhashable type', + ["f'{ {{}} }'", # dict in a set + ]) + + def test_compile_time_concat(self): + x = 'def' + self.assertEqual('abc' f'## {x}ghi', 'abc## defghi') + self.assertEqual('abc' f'{x}' 'ghi', 'abcdefghi') + self.assertEqual('abc' f'{x}' 'gh' f'i{x:4}', 'abcdefghidef ') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{x' f'{x}', '{xdef') + self.assertEqual('{x}' f'{x}', '{x}def') + self.assertEqual('{{x}}' f'{x}', '{{x}}def') + self.assertEqual('{{x' f'{x}', '{{xdef') + self.assertEqual('x}}' f'{x}', 'x}}def') + self.assertEqual(f'{x}' 'x}}', 'defx}}') + self.assertEqual(f'{x}' '', 'def') + self.assertEqual('' f'{x}' '', 'def') + self.assertEqual('' f'{x}', 'def') + self.assertEqual(f'{x}' '2', 'def2') + self.assertEqual('1' f'{x}' '2', '1def2') + self.assertEqual('1' f'{x}', '1def') + self.assertEqual(f'{x}' f'-{x}', 'def-def') + self.assertEqual('' f'', '') + self.assertEqual('' f'' '', '') + self.assertEqual('' f'' '' f'', '') + self.assertEqual(f'', '') + self.assertEqual(f'' '', '') + self.assertEqual(f'' '' f'', '') + self.assertEqual(f'' '' f'' '', '') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3' f'}'", # can't concat to get a valid f-string + ]) + + def test_comments(self): + # These aren't comments, since they're in strings. + d = {'#': 'hash'} + self.assertEqual(f'{"#"}', '#') + self.assertEqual(f'{d["#"]}', 'hash') + + self.assertAllRaise(SyntaxError, "f-string cannot include '#'", + ["f'{1#}'", # error because the expression becomes "(1#)" + "f'{3(#)}'", + ]) + + def test_many_expressions(self): + # Create a string with many expressions in it. Note that + # because we have a space in here as a literal, we're actually + # going to use twice as many ast nodes: one for each literal + # plus one for each expression. + def build_fstr(n, extra=''): + return "f'" + ('{x} ' * n) + extra + "'" + + x = 'X' + width = 1 + + # Test around 256. + for i in range(250, 260): + self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + + # Test concatenating 2 largs fstrings. + self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) + + s = build_fstr(253, '{x:{width}} ') + self.assertEqual(eval(s), (x+' ')*254) + + # Test lots of expressions and constants, concatenated. + s = "f'{1}' 'x' 'y'" * 1024 + self.assertEqual(eval(s), '1xy' * 1024) + + def test_format_specifier_expressions(self): + width = 10 + precision = 4 + value = decimal.Decimal('12.34567') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'{10:#{1}0x}', ' 0xa') + self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') + self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') + self.assertEqual(f'{-10:{"-"}#{1}0{"x"}}', ' -0xa') + self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa') + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["""f'{"s"!r{":10"}}'""", + + # This looks like a nested format spec. + ]) + + self.assertAllRaise(SyntaxError, "invalid syntax", + [# Invalid sytax inside a nested spec. + "f'{4:{/5}}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", + [# Can't nest format specifiers. + "f'result: {value:{width:{0}}.{precision:1}}'", + ]) + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + [# No expansion inside conversion or for + # the : or ! itself. + """f'{"s"!{"r"}}'""", + ]) + + def test_side_effect_order(self): + class X: + def __init__(self): + self.i = 0 + def __format__(self, spec): + self.i += 1 + return str(self.i) + + x = X() + self.assertEqual(f'{x} {x}', '1 2') + + def test_missing_expression(self): + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + ["f'{}'", + "f'{ }'" + "f' {} '", + "f'{!r}'", + "f'{ !r}'", + "f'{10:{ }}'", + "f' { } '", + r"f'{\n}'", + r"f'{\n \n}'", + ]) + + def test_parens_in_expressions(self): + self.assertEqual(f'{3,}', '(3,)') + + # Add these because when an expression is evaluated, parens + # are added around it. But we shouldn't go from an invalid + # expression to a valid one. The added parens are just + # supposed to allow whitespace (including newlines). + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'{,}'", + "f'{,}'", # this is (,), which is an error + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3)+(4}'", + ]) + + self.assertAllRaise(SyntaxError, 'EOL while scanning string literal', + ["f'{\n}'", + ]) + + def test_newlines_in_expressions(self): + self.assertEqual(f'{0}', '0') + self.assertEqual(f'{0\n}', '0') + self.assertEqual(f'{0\r}', '0') + self.assertEqual(f'{\n0\n}', '0') + self.assertEqual(f'{\r0\r}', '0') + self.assertEqual(f'{\n0\r}', '0') + self.assertEqual(f'{\n0}', '0') + self.assertEqual(f'{3+\n4}', '7') + self.assertEqual(f'{3+\\\n4}', '7') + self.assertEqual(rf'''{3+ +4}''', '7') + self.assertEqual(f'''{3+\ +4}''', '7') + + self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed', + [r"f'{\n}'", + ]) + + def test_lambda(self): + x = 5 + self.assertEqual(f'{(lambda y:x*y)("8")!r}', "'88888'") + self.assertEqual(f'{(lambda y:x*y)("8")!r:10}', "'88888' ") + self.assertEqual(f'{(lambda y:x*y)("8"):10}', "88888 ") + + # lambda doesn't work without parens, because the colon + # makes the parser think it's a format_spec + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["f'{lambda x:x}'", + ]) + + def test_yield(self): + # Not terribly useful, but make sure the yield turns + # a function into a generator + def fn(y): + f'y:{yield y*2}' + + g = fn(4) + self.assertEqual(next(g), 8) + + def test_yield_send(self): + def fn(x): + yield f'x:{yield (lambda i: x * i)}' + + g = fn(10) + the_lambda = next(g) + self.assertEqual(the_lambda(4), 40) + self.assertEqual(g.send('string'), 'x:string') + + def test_expressions_with_triple_quoted_strings(self): + self.assertEqual(f"{'''x'''}", 'x') + self.assertEqual(f"{'''eric's'''}", "eric's") + self.assertEqual(f'{"""eric\'s"""}', "eric's") + self.assertEqual(f"{'''eric\"s'''}", 'eric"s') + self.assertEqual(f'{"""eric"s"""}', 'eric"s') + + # Test concatenation within an expression + self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"x" """eric"s"""}', 'xeric"s') + self.assertEqual(f'{"""eric"s""" "y"}', 'eric"sy') + self.assertEqual(f'{"""x""" """eric"s""" "y"}', 'xeric"sy') + self.assertEqual(f'{"""x""" """eric"s""" """y"""}', 'xeric"sy') + self.assertEqual(f'{r"""x""" """eric"s""" """y"""}', 'xeric"sy') + + def test_multiple_vars(self): + x = 98 + y = 'abc' + self.assertEqual(f'{x}{y}', '98abc') + + self.assertEqual(f'X{x}{y}', 'X98abc') + self.assertEqual(f'{x}X{y}', '98Xabc') + self.assertEqual(f'{x}{y}X', '98abcX') + + self.assertEqual(f'X{x}Y{y}', 'X98Yabc') + self.assertEqual(f'X{x}{y}Y', 'X98abcY') + self.assertEqual(f'{x}X{y}Y', '98XabcY') + + self.assertEqual(f'X{x}Y{y}Z', 'X98YabcZ') + + def test_closure(self): + def outer(x): + def inner(): + return f'x:{x}' + return inner + + self.assertEqual(outer('987')(), 'x:987') + self.assertEqual(outer(7)(), 'x:7') + + def test_arguments(self): + y = 2 + def f(x, width): + return f'x={x*y:{width}}' + + self.assertEqual(f('foo', 10), 'x=foofoo ') + x = 'bar' + self.assertEqual(f(10, 10), 'x= 20') + + def test_locals(self): + value = 123 + self.assertEqual(f'v:{value}', 'v:123') + + def test_missing_variable(self): + with self.assertRaises(NameError): + f'v:{value}' + + def test_missing_format_spec(self): + class O: + def __format__(self, spec): + if not spec: + return '*' + return spec + + self.assertEqual(f'{O():x}', 'x') + self.assertEqual(f'{O()}', '*') + self.assertEqual(f'{O():}', '*') + + self.assertEqual(f'{3:}', '3') + self.assertEqual(f'{3!s:}', '3') + + def test_global(self): + self.assertEqual(f'g:{a_global}', 'g:global variable') + self.assertEqual(f'g:{a_global!r}', "g:'global variable'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:global variable l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'global variable'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:global variable l:'local variable'") + + self.assertIn("module 'unittest' from", f'{unittest}') + + def test_shadowed_global(self): + a_global = 'really a local' + self.assertEqual(f'g:{a_global}', 'g:really a local') + self.assertEqual(f'g:{a_global!r}', "g:'really a local'") + + a_local = 'local variable' + self.assertEqual(f'g:{a_global} l:{a_local}', + 'g:really a local l:local variable') + self.assertEqual(f'g:{a_global!r}', + "g:'really a local'") + self.assertEqual(f'g:{a_global} l:{a_local!r}', + "g:really a local l:'local variable'") + + def test_call(self): + def foo(x): + return 'x=' + str(x) + + self.assertEqual(f'{foo(10)}', 'x=10') + + def test_nested_fstrings(self): + y = 5 + self.assertEqual(f'{f"{0}"*3}', '000') + self.assertEqual(f'{f"{y}"*3}', '555') + self.assertEqual(f'{f"{\'x\'}"*3}', 'xxx') + + self.assertEqual(f"{r'x' f'{\"s\"}'}", 'xs') + self.assertEqual(f"{r'x'rf'{\"s\"}'}", 'xs') + + def test_invalid_string_prefixes(self): + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + ["fu''", + "uf''", + "Fu''", + "fU''", + "Uf''", + "uF''", + "ufr''", + "urf''", + "fur''", + "fru''", + "rfu''", + "ruf''", + "FUR''", + "Fur''", + ]) + + def test_leading_trailing_spaces(self): + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{ 3}', '3') + self.assertEqual(f'{\t3}', '3') + self.assertEqual(f'{\t\t3}', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3 }', '3') + self.assertEqual(f'{3\t}', '3') + self.assertEqual(f'{3\t\t}', '3') + + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}', + 'expr={1: 2}') + self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }', + 'expr={1: 2}') + + def test_character_name(self): + self.assertEqual(f'{4}\N{GREEK CAPITAL LETTER DELTA}{3}', + '4\N{GREEK CAPITAL LETTER DELTA}3') + self.assertEqual(f'{{}}\N{GREEK CAPITAL LETTER DELTA}{3}', + '{}\N{GREEK CAPITAL LETTER DELTA}3') + + def test_not_equal(self): + # There's a special test for this because there's a special + # case in the f-string parser to look for != as not ending an + # expression. Normally it would, while looking for !s or !r. + + self.assertEqual(f'{3!=4}', 'True') + self.assertEqual(f'{3!=4:}', 'True') + self.assertEqual(f'{3!=4!s}', 'True') + self.assertEqual(f'{3!=4!s:.3}', 'Tru') + + def test_conversions(self): + self.assertEqual(f'{3.14:10.10}', ' 3.14') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + + self.assertEqual(f'{"a"}', 'a') + self.assertEqual(f'{"a"!r}', "'a'") + self.assertEqual(f'{"a"!a}', "'a'") + + # Not a conversion. + self.assertEqual(f'{"a!r"}', "a!r") + + # Not a conversion, but show that ! is allowed in a format spec. + self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!') + + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"}', '\u0394') + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!r}', "'\u0394'") + self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!a}', "'\\u0394'") + + self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character', + ["f'{3!g}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!A}'", + "f'{3!!}'", + "f'{3!:}'", + "f'{3!\N{GREEK CAPITAL LETTER DELTA}}'", + "f'{3! s}'", # no space before conversion char + "f'{x!\\x00:.<10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{x!s{y}}'", + "f'{3!ss}'", + "f'{3!ss:}'", + "f'{3!ss:s}'", + ]) + + def test_assignment(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["f'' = 3", + "f'{0}' = x", + "f'{x}' = x", + ]) + + def test_del(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + ["del f''", + "del '' f''", + ]) + + def test_mismatched_braces(self): + self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed", + ["f'{{}'", + "f'{{}}}'", + "f'}'", + "f'x}'", + "f'x}x'", + + # Can't have { or } in a format spec. + "f'{3:}>10}'", + r"f'{3:\\}>10}'", + "f'{3:}}>10}'", + ]) + + self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + ["f'{3:{{>10}'", + "f'{3'", + "f'{3!'", + "f'{3:'", + "f'{3!s'", + "f'{3!s:'", + "f'{3!s:3'", + "f'x{'", + "f'x{x'", + "f'{3:s'", + "f'{{{'", + "f'{{}}{'", + "f'{'", + ]) + + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{3:\\{>10}'", + ]) + + # But these are just normal strings. + self.assertEqual(f'{"{"}', '{') + self.assertEqual(f'{"}"}', '}') + self.assertEqual(f'{3:{"}"}>10}', '}}}}}}}}}3') + self.assertEqual(f'{2:{"{"}>10}', '{{{{{{{{{2') + + def test_if_conditional(self): + # There's special logic in compile.c to test if the + # conditional for an if (and while) are constants. Exercise + # that code. + + def test_fstring(x, expected): + flag = 0 + if f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_empty(x, expected): + flag = 0 + if '' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + def test_concat_non_empty(x, expected): + flag = 0 + if ' ' f'{x}': + flag = 1 + else: + flag = 2 + self.assertEqual(flag, expected) + + test_fstring('', 2) + test_fstring(' ', 1) + + test_concat_empty('', 2) + test_concat_empty(' ', 1) + + test_concat_non_empty('', 1) + test_concat_non_empty(' ', 1) + + def test_empty_format_specifier(self): + x = 'test' + self.assertEqual(f'{x}', 'test') + self.assertEqual(f'{x:}', 'test') + self.assertEqual(f'{x!s:}', 'test') + self.assertEqual(f'{x!r:}', "'test'") + + def test_str_format_differences(self): + d = {'a': 'string', + 0: 'integer', + } + a = 0 + self.assertEqual(f'{d[0]}', 'integer') + self.assertEqual(f'{d["a"]}', 'string') + self.assertEqual(f'{d[a]}', 'integer') + self.assertEqual('{d[a]}'.format(d=d), 'string') + self.assertEqual('{d[0]}'.format(d=d), 'integer') + + def test_invalid_expressions(self): + self.assertAllRaise(SyntaxError, 'invalid syntax', + [r"f'{a[4)}'", + r"f'{a(4]}'", + ]) + + def test_loop(self): + for i in range(1000): + self.assertEqual(f'i:{i}', 'i:' + str(i)) + + def test_dict(self): + d = {'"': 'dquote', + "'": 'squote', + 'foo': 'bar', + } + self.assertEqual(f'{d["\'"]}', 'squote') + self.assertEqual(f"{d['\"']}", 'dquote') + + self.assertEqual(f'''{d["'"]}''', 'squote') + self.assertEqual(f"""{d['"']}""", 'dquote') + + self.assertEqual(f'{d["foo"]}', 'bar') + self.assertEqual(f"{d['foo']}", 'bar') + self.assertEqual(f'{d[\'foo\']}', 'bar') + self.assertEqual(f"{d[\"foo\"]}", 'bar') + + def test_escaped_quotes(self): + d = {'"': 'a', + "'": 'b'} + + self.assertEqual(fr"{d['\"']}", 'a') + self.assertEqual(fr'{d["\'"]}', 'b') + self.assertEqual(fr"{'\"'}", '"') + self.assertEqual(fr'{"\'"}', "'") + self.assertEqual(f'{"\\"3"}', '"3') + + self.assertAllRaise(SyntaxError, 'f-string: unterminated string', + [r'''f'{"""\\}' ''', # Backslash at end of expression + ]) + self.assertAllRaise(SyntaxError, 'unexpected character after line continuation', + [r"rf'{3\}'", + ]) + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index a2c76f5..46f7a3e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,11 @@ Core and Builtins argument list of a function declaration. For example, "def f(*, a = 3,): pass" is now legal. Patch from Mark Dickinson. +- Issue #24965: Implement PEP 498 "Literal String Interpolation". This + allows you to embed expressions inside f-strings, which are + converted to normal strings at run time. Given x=3, then + f'value={x}' == 'value=3'. Patch by Eric V. Smith. + Library ------- diff --git a/Parser/Python.asdl b/Parser/Python.asdl index cd0832d..22775c6 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -71,6 +71,8 @@ module Python | Call(expr func, expr* args, keyword* keywords) | Num(object n) -- a number as a PyObject. | Str(string s) -- need to specify raw, unicode, etc? + | FormattedValue(expr value, int? conversion, expr? format_spec) + | JoinedStr(expr* values) | Bytes(bytes s) | NameConstant(singleton value) | Ellipsis diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5046fa5..2369be4 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1477,17 +1477,19 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) nonascii = 0; if (is_potential_identifier_start(c)) { /* Process b"", r"", u"", br"" and rb"" */ - int saw_b = 0, saw_r = 0, saw_u = 0; + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0; while (1) { - if (!(saw_b || saw_u) && (c == 'b' || c == 'B')) + if (!(saw_b || saw_u || saw_f) && (c == 'b' || c == 'B')) saw_b = 1; /* Since this is a backwards compatibility support literal we don't want to support it in arbitrary order like byte literals. */ - else if (!(saw_b || saw_u || saw_r) && (c == 'u' || c == 'U')) + else if (!(saw_b || saw_u || saw_r || saw_f) && (c == 'u' || c == 'U')) saw_u = 1; /* ur"" and ru"" are not supported */ else if (!(saw_r || saw_u) && (c == 'r' || c == 'R')) saw_r = 1; + else if (!(saw_f || saw_b || saw_u) && (c == 'f' || c == 'F')) + saw_f = 1; else break; c = tok_nextc(tok); diff --git a/Python/Python-ast.c b/Python/Python-ast.c index fd7f17e..a2e9816 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -285,6 +285,18 @@ _Py_IDENTIFIER(s); static char *Str_fields[]={ "s", }; +static PyTypeObject *FormattedValue_type; +_Py_IDENTIFIER(conversion); +_Py_IDENTIFIER(format_spec); +static char *FormattedValue_fields[]={ + "value", + "conversion", + "format_spec", +}; +static PyTypeObject *JoinedStr_type; +static char *JoinedStr_fields[]={ + "values", +}; static PyTypeObject *Bytes_type; static char *Bytes_fields[]={ "s", @@ -917,6 +929,11 @@ static int init_types(void) if (!Num_type) return 0; Str_type = make_type("Str", expr_type, Str_fields, 1); if (!Str_type) return 0; + FormattedValue_type = make_type("FormattedValue", expr_type, + FormattedValue_fields, 3); + if (!FormattedValue_type) return 0; + JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1); + if (!JoinedStr_type) return 0; Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1); if (!Bytes_type) return 0; NameConstant_type = make_type("NameConstant", expr_type, @@ -2063,6 +2080,42 @@ Str(string s, int lineno, int col_offset, PyArena *arena) } expr_ty +FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno, + int col_offset, PyArena *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for FormattedValue"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = FormattedValue_kind; + p->v.FormattedValue.value = value; + p->v.FormattedValue.conversion = conversion; + p->v.FormattedValue.format_spec = format_spec; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty +JoinedStr(asdl_seq * values, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = JoinedStr_kind; + p->v.JoinedStr.values = values; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Bytes(bytes s, int lineno, int col_offset, PyArena *arena) { expr_ty p; @@ -3161,6 +3214,34 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; + case FormattedValue_kind: + result = PyType_GenericNew(FormattedValue_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_expr(o->v.FormattedValue.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_int(o->v.FormattedValue.conversion); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_conversion, value) == -1) + goto failed; + Py_DECREF(value); + value = ast2obj_expr(o->v.FormattedValue.format_spec); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1) + goto failed; + Py_DECREF(value); + break; + case JoinedStr_kind: + result = PyType_GenericNew(JoinedStr_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_list(o->v.JoinedStr.values, ast2obj_expr); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_values, value) == -1) + goto failed; + Py_DECREF(value); + break; case Bytes_kind: result = PyType_GenericNew(Bytes_type, NULL, NULL); if (!result) goto failed; @@ -6022,6 +6103,86 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)FormattedValue_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + expr_ty value; + int conversion; + expr_ty format_spec; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); + return 1; + } + if (exists_not_none(obj, &PyId_conversion)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_conversion); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &conversion, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + conversion = 0; + } + if (exists_not_none(obj, &PyId_format_spec)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_format_spec); + if (tmp == NULL) goto failed; + res = obj2ast_expr(tmp, &format_spec, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + format_spec = NULL; + } + *out = FormattedValue(value, conversion, format_spec, lineno, + col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } + isinstance = PyObject_IsInstance(obj, (PyObject*)JoinedStr_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + asdl_seq* values; + + if (_PyObject_HasAttrId(obj, &PyId_values)) { + int res; + Py_ssize_t len; + Py_ssize_t i; + tmp = _PyObject_GetAttrId(obj, &PyId_values); + if (tmp == NULL) goto failed; + if (!PyList_Check(tmp)) { + PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); + goto failed; + } + len = PyList_GET_SIZE(tmp); + values = _Py_asdl_seq_new(len, arena); + if (values == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(values, i, value); + } + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr"); + return 1; + } + *out = JoinedStr(values, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type); if (isinstance == -1) { return 1; @@ -7319,6 +7480,10 @@ PyInit__ast(void) if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL; if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return NULL; if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return NULL; + if (PyDict_SetItemString(d, "FormattedValue", + (PyObject*)FormattedValue_type) < 0) return NULL; + if (PyDict_SetItemString(d, "JoinedStr", (PyObject*)JoinedStr_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return NULL; if (PyDict_SetItemString(d, "NameConstant", (PyObject*)NameConstant_type) < diff --git a/Python/ast.c b/Python/ast.c index 1f7ddfc..735424b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -257,6 +257,14 @@ validate_expr(expr_ty exp, expr_context_ty ctx) } return 1; } + case JoinedStr_kind: + return validate_exprs(exp->v.JoinedStr.values, Load, 0); + case FormattedValue_kind: + if (validate_expr(exp->v.FormattedValue.value, Load) == 0) + return 0; + if (exp->v.FormattedValue.format_spec) + return validate_expr(exp->v.FormattedValue.format_spec, Load); + return 1; case Bytes_kind: { PyObject *b = exp->v.Bytes.s; if (!PyBytes_CheckExact(b)) { @@ -535,9 +543,7 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, int); static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); static PyObject *parsenumber(struct compiling *, const char *); -static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode); -static PyObject *parsestrplus(struct compiling *, const node *n, - int *bytesmode); +static expr_ty parsestrplus(struct compiling *, const node *n); #define COMP_GENEXP 0 #define COMP_LISTCOMP 1 @@ -986,6 +992,8 @@ set_context(struct compiling *c, expr_ty e, expr_context_ty ctx, const node *n) case Num_kind: case Str_kind: case Bytes_kind: + case JoinedStr_kind: + case FormattedValue_kind: expr_name = "literal"; break; case NameConstant_kind: @@ -2001,7 +2009,6 @@ ast_for_atom(struct compiling *c, const node *n) | '...' | 'None' | 'True' | 'False' */ node *ch = CHILD(n, 0); - int bytesmode = 0; switch (TYPE(ch)) { case NAME: { @@ -2023,7 +2030,7 @@ ast_for_atom(struct compiling *c, const node *n) return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); } case STRING: { - PyObject *str = parsestrplus(c, n, &bytesmode); + expr_ty str = parsestrplus(c, n); if (!str) { const char *errtype = NULL; if (PyErr_ExceptionMatches(PyExc_UnicodeError)) @@ -2050,14 +2057,7 @@ ast_for_atom(struct compiling *c, const node *n) } return NULL; } - if (PyArena_AddPyObject(c->c_arena, str) < 0) { - Py_DECREF(str); - return NULL; - } - if (bytesmode) - return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); - else - return Str(str, LINENO(n), n->n_col_offset, c->c_arena); + return str; } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); @@ -4002,12 +4002,838 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons return v; } -/* s is a Python string literal, including the bracketing quote characters, - * and r &/or b prefixes (if any), and embedded escape sequences (if any). - * parsestr parses it, and returns the decoded Python string object. - */ +/* Compile this expression in to an expr_ty. We know that we can + temporarily modify the character before the start of this string + (it's '{'), and we know we can temporarily modify the character + after this string (it is a '}'). Leverage this to create a + sub-string with enough room for us to add parens around the + expression. This is to allow strings with embedded newlines, for + example. */ +static expr_ty +fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, + Py_ssize_t expr_end, PyArena *arena) +{ + PyCompilerFlags cf; + mod_ty mod; + char *utf_expr; + Py_ssize_t i; + int all_whitespace; + PyObject *sub = NULL; + + /* We only decref sub if we allocated it with a PyUnicode_Substring. + decref_sub records that. */ + int decref_sub = 0; + + assert(str); + + /* If the substring is all whitespace, it's an error. We need to + catch this here, and not when we call PyParser_ASTFromString, + because turning the expression '' in to '()' would go from + being invalid to valid. */ + /* Note that this code says an empty string is all + whitespace. That's important. There's a test for it: f'{}'. */ + all_whitespace = 1; + for (i = expr_start; i < expr_end; i++) { + if (!Py_UNICODE_ISSPACE(PyUnicode_READ_CHAR(str, i))) { + all_whitespace = 0; + break; + } + } + if (all_whitespace) { + PyErr_SetString(PyExc_SyntaxError, "f-string: empty expression " + "not allowed"); + goto error; + } + + /* If the substring will be the entire source string, we can't use + PyUnicode_Substring, since it will return another reference to + our original string. Because we're modifying the string in + place, that's a no-no. So, detect that case and just use our + string directly. */ + + if (expr_start-1 == 0 && expr_end+1 == PyUnicode_GET_LENGTH(str)) { + /* No need to actually remember these characters, because we + know they must be braces. */ + assert(PyUnicode_ReadChar(str, 0) == '{'); + assert(PyUnicode_ReadChar(str, expr_end-expr_start+1) == '}'); + sub = str; + } else { + /* Create a substring object. It must be a new object, with + refcount==1, so that we can modify it. */ + sub = PyUnicode_Substring(str, expr_start-1, expr_end+1); + if (!sub) + goto error; + assert(sub != str); /* Make sure it's a new string. */ + decref_sub = 1; /* Remember to deallocate it on error. */ + } + + if (PyUnicode_WriteChar(sub, 0, '(') < 0 || + PyUnicode_WriteChar(sub, expr_end-expr_start+1, ')') < 0) + goto error; + + cf.cf_flags = PyCF_ONLY_AST; + + /* No need to free the memory returned here: it's managed by the + string. */ + utf_expr = PyUnicode_AsUTF8(sub); + if (!utf_expr) + goto error; + mod = PyParser_ASTFromString(utf_expr, "", + Py_eval_input, &cf, arena); + if (!mod) + goto error; + if (sub != str) + /* Clear instead of decref in case we ever modify this code to change + the error handling: this is safest because the XDECREF won't try + and decref it when it's NULL. */ + /* No need to restore the chars in sub, since we know it's getting + ready to get deleted (refcount must be 1, since we got a new string + in PyUnicode_Substring). */ + Py_CLEAR(sub); + else { + assert(!decref_sub); + /* Restore str, which we earlier modified directly. */ + if (PyUnicode_WriteChar(str, 0, '{') < 0 || + PyUnicode_WriteChar(str, expr_end-expr_start+1, '}') < 0) + goto error; + } + return mod->v.Expression.body; + +error: + /* Only decref sub if it was the result of a call to SubString. */ + if (decref_sub) + Py_XDECREF(sub); + return NULL; +} + +/* Return -1 on error. + + Return 0 if we reached the end of the literal. + + Return 1 if we haven't reached the end of the literal, but we want + the caller to process the literal up to this point. Used for + doubled braces. +*/ +static int +fstring_find_literal(PyObject *str, Py_ssize_t *ofs, PyObject **literal, + int recurse_lvl, struct compiling *c, const node *n) +{ + /* Get any literal string. It ends when we hit an un-doubled brace, or the + end of the string. */ + + Py_ssize_t literal_start, literal_end; + int result = 0; + + enum PyUnicode_Kind kind = PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + + assert(*literal == NULL); + + literal_start = *ofs; + for (; *ofs < PyUnicode_GET_LENGTH(str); *ofs += 1) { + Py_UCS4 ch = PyUnicode_READ(kind, data, *ofs); + if (ch == '{' || ch == '}') { + /* Check for doubled braces, but only at the top level. If + we checked at every level, then f'{0:{3}}' would fail + with the two closing braces. */ + if (recurse_lvl == 0) { + if (*ofs + 1 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs + 1) == ch) { + /* We're going to tell the caller that the literal ends + here, but that they should continue scanning. But also + skip over the second brace when we resume scanning. */ + literal_end = *ofs + 1; + *ofs += 2; + result = 1; + goto done; + } + + /* Where a single '{' is the start of a new expression, a + single '}' is not allowed. */ + if (ch == '}') { + ast_error(c, n, "f-string: single '}' is not allowed"); + return -1; + } + } + + /* We're either at a '{', which means we're starting another + expression; or a '}', which means we're at the end of this + f-string (for a nested format_spec). */ + break; + } + } + literal_end = *ofs; + + assert(*ofs == PyUnicode_GET_LENGTH(str) || + PyUnicode_READ(kind, data, *ofs) == '{' || + PyUnicode_READ(kind, data, *ofs) == '}'); +done: + if (literal_start != literal_end) { + *literal = PyUnicode_Substring(str, literal_start, literal_end); + if (!*literal) + return -1; + } + + return result; +} + +/* Forward declaration because parsing is recursive. */ +static expr_ty +fstring_parse(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n); + +/* Parse the f-string str, starting at ofs. We know *ofs starts an + expression (so it must be a '{'). Returns the FormattedValue node, + which includes the expression, conversion character, and + format_spec expression. + + Note that I don't do a perfect job here: I don't make sure that a + closing brace doesn't match an opening paren, for example. It + doesn't need to error on all invalid expressions, just correctly + find the end of all valid ones. Any errors inside the expression + will be caught when we parse it later. */ +static int +fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + expr_ty *expression, struct compiling *c, const node *n) +{ + /* Return -1 on error, else 0. */ + + Py_ssize_t expr_start; + Py_ssize_t expr_end; + expr_ty simple_expression; + expr_ty format_spec = NULL; /* Optional format specifier. */ + Py_UCS4 conversion = -1; /* The conversion char. -1 if not specified. */ + + enum PyUnicode_Kind kind = PyUnicode_KIND(str); + void *data = PyUnicode_DATA(str); + + /* 0 if we're not in a string, else the quote char we're trying to + match (single or double quote). */ + Py_UCS4 quote_char = 0; + + /* If we're inside a string, 1=normal, 3=triple-quoted. */ + int string_type = 0; + + /* Keep track of nesting level for braces/parens/brackets in + expressions. */ + Py_ssize_t nested_depth = 0; + + /* Can only nest one level deep. */ + if (recurse_lvl >= 2) { + ast_error(c, n, "f-string: expressions nested too deeply"); + return -1; + } + + /* The first char must be a left brace, or we wouldn't have gotten + here. Skip over it. */ + assert(PyUnicode_READ(kind, data, *ofs) == '{'); + *ofs += 1; + + expr_start = *ofs; + for (; *ofs < PyUnicode_GET_LENGTH(str); *ofs += 1) { + Py_UCS4 ch; + + /* Loop invariants. */ + assert(nested_depth >= 0); + assert(*ofs >= expr_start); + if (quote_char) + assert(string_type == 1 || string_type == 3); + else + assert(string_type == 0); + + ch = PyUnicode_READ(kind, data, *ofs); + if (quote_char) { + /* We're inside a string. See if we're at the end. */ + /* This code needs to implement the same non-error logic + as tok_get from tokenizer.c, at the letter_quote + label. To actually share that code would be a + nightmare. But, it's unlikely to change and is small, + so duplicate it here. Note we don't need to catch all + of the errors, since they'll be caught when parsing the + expression. We just need to match the non-error + cases. Thus we can ignore \n in single-quoted strings, + for example. Or non-terminated strings. */ + if (ch == quote_char) { + /* Does this match the string_type (single or triple + quoted)? */ + if (string_type == 3) { + if (*ofs+2 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == ch && + PyUnicode_READ(kind, data, *ofs+2) == ch) { + /* We're at the end of a triple quoted string. */ + *ofs += 2; + string_type = 0; + quote_char = 0; + continue; + } + } else { + /* We're at the end of a normal string. */ + quote_char = 0; + string_type = 0; + continue; + } + } + /* We're inside a string, and not finished with the + string. If this is a backslash, skip the next char (it + might be an end quote that needs skipping). Otherwise, + just consume this character normally. */ + if (ch == '\\' && *ofs+1 < PyUnicode_GET_LENGTH(str)) { + /* Just skip the next char, whatever it is. */ + *ofs += 1; + } + } else if (ch == '\'' || ch == '"') { + /* Is this a triple quoted string? */ + if (*ofs+2 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == ch && + PyUnicode_READ(kind, data, *ofs+2) == ch) { + string_type = 3; + *ofs += 2; + } else { + /* Start of a normal string. */ + string_type = 1; + } + /* Start looking for the end of the string. */ + quote_char = ch; + } else if (ch == '[' || ch == '{' || ch == '(') { + nested_depth++; + } else if (nested_depth != 0 && + (ch == ']' || ch == '}' || ch == ')')) { + nested_depth--; + } else if (ch == '#') { + /* Error: can't include a comment character, inside parens + or not. */ + ast_error(c, n, "f-string cannot include '#'"); + return -1; + } else if (nested_depth == 0 && + (ch == '!' || ch == ':' || ch == '}')) { + /* First, test for the special case of "!=". Since '=' is + not an allowed conversion character, nothing is lost in + this test. */ + if (ch == '!' && *ofs+1 < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ(kind, data, *ofs+1) == '=') + /* This isn't a conversion character, just continue. */ + continue; + + /* Normal way out of this loop. */ + break; + } else { + /* Just consume this char and loop around. */ + } + } + expr_end = *ofs; + /* If we leave this loop in a string or with mismatched parens, we + don't care. We'll get a syntax error when compiling the + expression. But, we can produce a better error message, so + let's just do that.*/ + if (quote_char) { + ast_error(c, n, "f-string: unterminated string"); + return -1; + } + if (nested_depth) { + ast_error(c, n, "f-string: mismatched '(', '{', or '['"); + return -1; + } + + /* Check for a conversion char, if present. */ + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + if (PyUnicode_READ(kind, data, *ofs) == '!') { + *ofs += 1; + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + + conversion = PyUnicode_READ(kind, data, *ofs); + *ofs += 1; + + /* Validate the conversion. */ + if (!(conversion == 's' || conversion == 'r' + || conversion == 'a')) { + ast_error(c, n, "f-string: invalid conversion character: " + "expected 's', 'r', or 'a'"); + return -1; + } + } + + /* Check for the format spec, if present. */ + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + if (PyUnicode_READ(kind, data, *ofs) == ':') { + *ofs += 1; + if (*ofs >= PyUnicode_GET_LENGTH(str)) + goto unexpected_end_of_string; + + /* Parse the format spec. */ + format_spec = fstring_parse(str, ofs, recurse_lvl+1, c, n); + if (!format_spec) + return -1; + } + + if (*ofs >= PyUnicode_GET_LENGTH(str) || + PyUnicode_READ(kind, data, *ofs) != '}') + goto unexpected_end_of_string; + + /* We're at a right brace. Consume it. */ + assert(*ofs < PyUnicode_GET_LENGTH(str)); + assert(PyUnicode_READ(kind, data, *ofs) == '}'); + *ofs += 1; + + /* Compile the expression. */ + simple_expression = fstring_expression_compile(str, expr_start, expr_end, + c->c_arena); + if (!simple_expression) + return -1; + + /* And now create the FormattedValue node that represents this entire + expression with the conversion and format spec. */ + *expression = FormattedValue(simple_expression, (int)conversion, + format_spec, LINENO(n), n->n_col_offset, + c->c_arena); + if (!*expression) + return -1; + + return 0; + +unexpected_end_of_string: + ast_error(c, n, "f-string: expecting '}'"); + return -1; +} + +/* Return -1 on error. + + Return 0 if we have a literal (possible zero length) and an + expression (zero length if at the end of the string. + + Return 1 if we have a literal, but no expression, and we want the + caller to call us again. This is used to deal with doubled + braces. + + When called multiple times on the string 'a{{b{0}c', this function + will return: + + 1. the literal 'a{' with no expression, and a return value + of 1. Despite the fact that there's no expression, the return + value of 1 means we're not finished yet. + + 2. the literal 'b' and the expression '0', with a return value of + 0. The fact that there's an expression means we're not finished. + + 3. literal 'c' with no expression and a return value of 0. The + combination of the return value of 0 with no expression means + we're finished. +*/ +static int +fstring_find_literal_and_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + PyObject **literal, expr_ty *expression, + struct compiling *c, const node *n) +{ + int result; + + assert(*literal == NULL && *expression == NULL); + + /* Get any literal string. */ + result = fstring_find_literal(str, ofs, literal, recurse_lvl, c, n); + if (result < 0) + goto error; + + assert(result == 0 || result == 1); + + if (result == 1) + /* We have a literal, but don't look at the expression. */ + return 1; + + assert(*ofs <= PyUnicode_GET_LENGTH(str)); + + if (*ofs >= PyUnicode_GET_LENGTH(str) || + PyUnicode_READ_CHAR(str, *ofs) == '}') + /* We're at the end of the string or the end of a nested + f-string: no expression. The top-level error case where we + expect to be at the end of the string but we're at a '}' is + handled later. */ + return 0; + + /* We must now be the start of an expression, on a '{'. */ + assert(*ofs < PyUnicode_GET_LENGTH(str) && + PyUnicode_READ_CHAR(str, *ofs) == '{'); + + if (fstring_find_expr(str, ofs, recurse_lvl, expression, c, n) < 0) + goto error; + + return 0; + +error: + Py_XDECREF(*literal); + *literal = NULL; + return -1; +} + +#define EXPRLIST_N_CACHED 64 + +typedef struct { + /* Incrementally build an array of expr_ty, so be used in an + asdl_seq. Cache some small but reasonably sized number of + expr_ty's, and then after that start dynamically allocating, + doubling the number allocated each time. Note that the f-string + f'{0}a{1}' contains 3 expr_ty's: 2 FormattedValue's, and one + Str for the literal 'a'. So you add expr_ty's about twice as + fast as you add exressions in an f-string. */ + + Py_ssize_t allocated; /* Number we've allocated. */ + Py_ssize_t size; /* Number we've used. */ + expr_ty *p; /* Pointer to the memory we're actually + using. Will point to 'data' until we + start dynamically allocating. */ + expr_ty data[EXPRLIST_N_CACHED]; +} ExprList; + +#ifdef NDEBUG +#define ExprList_check_invariants(l) +#else +static void +ExprList_check_invariants(ExprList *l) +{ + /* Check our invariants. Make sure this object is "live", and + hasn't been deallocated. */ + assert(l->size >= 0); + assert(l->p != NULL); + if (l->size <= EXPRLIST_N_CACHED) + assert(l->data == l->p); +} +#endif + +static void +ExprList_Init(ExprList *l) +{ + l->allocated = EXPRLIST_N_CACHED; + l->size = 0; + + /* Until we start allocating dynamically, p points to data. */ + l->p = l->data; + + ExprList_check_invariants(l); +} + +static int +ExprList_Append(ExprList *l, expr_ty exp) +{ + ExprList_check_invariants(l); + if (l->size >= l->allocated) { + /* We need to alloc (or realloc) the memory. */ + Py_ssize_t new_size = l->allocated * 2; + + /* See if we've ever allocated anything dynamically. */ + if (l->p == l->data) { + Py_ssize_t i; + /* We're still using the cached data. Switch to + alloc-ing. */ + l->p = PyMem_RawMalloc(sizeof(expr_ty) * new_size); + if (!l->p) + return -1; + /* Copy the cached data into the new buffer. */ + for (i = 0; i < l->size; i++) + l->p[i] = l->data[i]; + } else { + /* Just realloc. */ + expr_ty *tmp = PyMem_RawRealloc(l->p, sizeof(expr_ty) * new_size); + if (!tmp) { + PyMem_RawFree(l->p); + l->p = NULL; + return -1; + } + l->p = tmp; + } + + l->allocated = new_size; + assert(l->allocated == 2 * l->size); + } + + l->p[l->size++] = exp; + + ExprList_check_invariants(l); + return 0; +} + +static void +ExprList_Dealloc(ExprList *l) +{ + ExprList_check_invariants(l); + + /* If there's been an error, or we've never dynamically allocated, + do nothing. */ + if (!l->p || l->p == l->data) { + /* Do nothing. */ + } else { + /* We have dynamically allocated. Free the memory. */ + PyMem_RawFree(l->p); + } + l->p = NULL; + l->size = -1; +} + +static asdl_seq * +ExprList_Finish(ExprList *l, PyArena *arena) +{ + asdl_seq *seq; + + ExprList_check_invariants(l); + + /* Allocate the asdl_seq and copy the expressions in to it. */ + seq = _Py_asdl_seq_new(l->size, arena); + if (seq) { + Py_ssize_t i; + for (i = 0; i < l->size; i++) + asdl_seq_SET(seq, i, l->p[i]); + } + ExprList_Dealloc(l); + return seq; +} + +/* The FstringParser is designed to add a mix of strings and + f-strings, and concat them together as needed. Ultimately, it + generates an expr_ty. */ +typedef struct { + PyObject *last_str; + ExprList expr_list; +} FstringParser; + +#ifdef NDEBUG +#define FstringParser_check_invariants(state) +#else +static void +FstringParser_check_invariants(FstringParser *state) +{ + if (state->last_str) + assert(PyUnicode_CheckExact(state->last_str)); + ExprList_check_invariants(&state->expr_list); +} +#endif + +static void +FstringParser_Init(FstringParser *state) +{ + state->last_str = NULL; + ExprList_Init(&state->expr_list); + FstringParser_check_invariants(state); +} + +static void +FstringParser_Dealloc(FstringParser *state) +{ + FstringParser_check_invariants(state); + + Py_XDECREF(state->last_str); + ExprList_Dealloc(&state->expr_list); +} + +/* Make a Str node, but decref the PyUnicode object being added. */ +static expr_ty +make_str_node_and_del(PyObject **str, struct compiling *c, const node* n) +{ + PyObject *s = *str; + *str = NULL; + assert(PyUnicode_CheckExact(s)); + if (PyArena_AddPyObject(c->c_arena, s) < 0) { + Py_DECREF(s); + return NULL; + } + return Str(s, LINENO(n), n->n_col_offset, c->c_arena); +} + +/* Add a non-f-string (that is, a regular literal string). str is + decref'd. */ +static int +FstringParser_ConcatAndDel(FstringParser *state, PyObject *str) +{ + FstringParser_check_invariants(state); + + assert(PyUnicode_CheckExact(str)); + + if (PyUnicode_GET_LENGTH(str) == 0) { + Py_DECREF(str); + return 0; + } + + if (!state->last_str) { + /* We didn't have a string before, so just remember this one. */ + state->last_str = str; + } else { + /* Concatenate this with the previous string. */ + PyObject *temp = PyUnicode_Concat(state->last_str, str); + Py_DECREF(state->last_str); + Py_DECREF(str); + state->last_str = temp; + if (!temp) + return -1; + } + FstringParser_check_invariants(state); + return 0; +} + +/* Parse an f-string. The f-string is in str, starting at ofs, with no 'f' + or quotes. str is not decref'd, since we don't know if it's used elsewhere. + And if we're only looking at a part of a string, then decref'ing is + definitely not the right thing to do! */ +static int +FstringParser_ConcatFstring(FstringParser *state, PyObject *str, + Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n) +{ + FstringParser_check_invariants(state); + + /* Parse the f-string. */ + while (1) { + PyObject *literal = NULL; + expr_ty expression = NULL; + + /* If there's a zero length literal in front of the + expression, literal will be NULL. If we're at the end of + the f-string, expression will be NULL (unless result == 1, + see below). */ + int result = fstring_find_literal_and_expr(str, ofs, recurse_lvl, + &literal, &expression, + c, n); + if (result < 0) + return -1; + + /* Add the literal, if any. */ + if (!literal) { + /* Do nothing. Just leave last_str alone (and possibly + NULL). */ + } else if (!state->last_str) { + state->last_str = literal; + literal = NULL; + } else { + /* We have a literal, concatenate it. */ + assert(PyUnicode_GET_LENGTH(literal) != 0); + if (FstringParser_ConcatAndDel(state, literal) < 0) + return -1; + literal = NULL; + } + assert(!state->last_str || + PyUnicode_GET_LENGTH(state->last_str) != 0); + + /* We've dealt with the literal now. It can't be leaked on further + errors. */ + assert(literal == NULL); + + /* See if we should just loop around to get the next literal + and expression, while ignoring the expression this + time. This is used for un-doubling braces, as an + optimization. */ + if (result == 1) + continue; + + if (!expression) + /* We're done with this f-string. */ + break; + + /* We know we have an expression. Convert any existing string + to a Str node. */ + if (!state->last_str) { + /* Do nothing. No previous literal. */ + } else { + /* Convert the existing last_str literal to a Str node. */ + expr_ty str = make_str_node_and_del(&state->last_str, c, n); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + return -1; + } + + if (ExprList_Append(&state->expr_list, expression) < 0) + return -1; + } + + assert(*ofs <= PyUnicode_GET_LENGTH(str)); + + /* If recurse_lvl is zero, then we must be at the end of the + string. Otherwise, we must be at a right brace. */ + + if (recurse_lvl == 0 && *ofs < PyUnicode_GET_LENGTH(str)) { + ast_error(c, n, "f-string: unexpected end of string"); + return -1; + } + if (recurse_lvl != 0 && PyUnicode_READ_CHAR(str, *ofs) != '}') { + ast_error(c, n, "f-string: expecting '}'"); + return -1; + } + + FstringParser_check_invariants(state); + return 0; +} + +/* Convert the partial state reflected in last_str and expr_list to an + expr_ty. The expr_ty can be a Str, or a JoinedStr. */ +static expr_ty +FstringParser_Finish(FstringParser *state, struct compiling *c, + const node *n) +{ + asdl_seq *seq; + + FstringParser_check_invariants(state); + + /* If we're just a constant string with no expressions, return + that. */ + if(state->expr_list.size == 0) { + if (!state->last_str) { + /* Create a zero length string. */ + state->last_str = PyUnicode_FromStringAndSize(NULL, 0); + if (!state->last_str) + goto error; + } + return make_str_node_and_del(&state->last_str, c, n); + } + + /* Create a Str node out of last_str, if needed. It will be the + last node in our expression list. */ + if (state->last_str) { + expr_ty str = make_str_node_and_del(&state->last_str, c, n); + if (!str || ExprList_Append(&state->expr_list, str) < 0) + goto error; + } + /* This has already been freed. */ + assert(state->last_str == NULL); + + seq = ExprList_Finish(&state->expr_list, c->c_arena); + if (!seq) + goto error; + + /* If there's only one expression, return it. Otherwise, we need + to join them together. */ + if (seq->size == 1) + return seq->elements[0]; + + return JoinedStr(seq, LINENO(n), n->n_col_offset, c->c_arena); + +error: + FstringParser_Dealloc(state); + return NULL; +} + +/* Given an f-string (with no 'f' or quotes) that's in str starting at + ofs, parse it into an expr_ty. Return NULL on error. Does not + decref str. */ +static expr_ty +fstring_parse(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, + struct compiling *c, const node *n) +{ + FstringParser state; + + FstringParser_Init(&state); + if (FstringParser_ConcatFstring(&state, str, ofs, recurse_lvl, + c, n) < 0) { + FstringParser_Dealloc(&state); + return NULL; + } + + return FstringParser_Finish(&state, c, n); +} + +/* n is a Python string literal, including the bracketing quote + characters, and r, b, u, &/or f prefixes (if any), and embedded + escape sequences (if any). parsestr parses it, and returns the + decoded Python string object. If the string is an f-string, set + *fmode and return the unparsed string object. +*/ static PyObject * -parsestr(struct compiling *c, const node *n, int *bytesmode) +parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) { size_t len; const char *s = STR(n); @@ -4027,15 +4853,24 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) quote = *++s; rawmode = 1; } + else if (quote == 'f' || quote == 'F') { + quote = *++s; + *fmode = 1; + } else { break; } } } + if (*fmode && *bytesmode) { + PyErr_BadInternalCall(); + return NULL; + } if (quote != '\'' && quote != '\"') { PyErr_BadInternalCall(); return NULL; } + /* Skip the leading quote char. */ s++; len = strlen(s); if (len > INT_MAX) { @@ -4044,12 +4879,17 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) return NULL; } if (s[--len] != quote) { + /* Last quote char must match the first. */ PyErr_BadInternalCall(); return NULL; } if (len >= 4 && s[0] == quote && s[1] == quote) { + /* A triple quoted string. We've already skipped one quote at + the start and one at the end of the string. Now skip the + two at the start. */ s += 2; len -= 2; + /* And check that the last two match. */ if (s[--len] != quote || s[--len] != quote) { PyErr_BadInternalCall(); return NULL; @@ -4088,51 +4928,84 @@ parsestr(struct compiling *c, const node *n, int *bytesmode) } } return PyBytes_DecodeEscape(s, len, NULL, 1, - need_encoding ? c->c_encoding : NULL); + need_encoding ? c->c_encoding : NULL); } -/* Build a Python string object out of a STRING+ atom. This takes care of - * compile-time literal catenation, calling parsestr() on each piece, and - * pasting the intermediate results together. - */ -static PyObject * -parsestrplus(struct compiling *c, const node *n, int *bytesmode) +/* Accepts a STRING+ atom, and produces an expr_ty node. Run through + each STRING atom, and process it as needed. For bytes, just + concatenate them together, and the result will be a Bytes node. For + normal strings and f-strings, concatenate them together. The result + will be a Str node if there were no f-strings; a FormattedValue + node if there's just an f-string (with no leading or trailing + literals), or a JoinedStr node if there are multiple f-strings or + any literals involved. */ +static expr_ty +parsestrplus(struct compiling *c, const node *n) { - PyObject *v; + int bytesmode = 0; + PyObject *bytes_str = NULL; int i; - REQ(CHILD(n, 0), STRING); - v = parsestr(c, CHILD(n, 0), bytesmode); - if (v != NULL) { - /* String literal concatenation */ - for (i = 1; i < NCH(n); i++) { - PyObject *s; - int subbm = 0; - s = parsestr(c, CHILD(n, i), &subbm); - if (s == NULL) - goto onError; - if (*bytesmode != subbm) { - ast_error(c, n, "cannot mix bytes and nonbytes literals"); - Py_DECREF(s); - goto onError; - } - if (PyBytes_Check(v) && PyBytes_Check(s)) { - PyBytes_ConcatAndDel(&v, s); - if (v == NULL) - goto onError; - } - else { - PyObject *temp = PyUnicode_Concat(v, s); - Py_DECREF(s); - Py_DECREF(v); - v = temp; - if (v == NULL) - goto onError; + + FstringParser state; + FstringParser_Init(&state); + + for (i = 0; i < NCH(n); i++) { + int this_bytesmode = 0; + int this_fmode = 0; + PyObject *s; + + REQ(CHILD(n, i), STRING); + s = parsestr(c, CHILD(n, i), &this_bytesmode, &this_fmode); + if (!s) + goto error; + + /* Check that we're not mixing bytes with unicode. */ + if (i != 0 && bytesmode != this_bytesmode) { + ast_error(c, n, "cannot mix bytes and nonbytes literals"); + Py_DECREF(s); + goto error; + } + bytesmode = this_bytesmode; + + assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s)); + + if (bytesmode) { + /* For bytes, concat as we go. */ + if (i == 0) { + /* First time, just remember this value. */ + bytes_str = s; + } else { + PyBytes_ConcatAndDel(&bytes_str, s); + if (!bytes_str) + goto error; } + } else if (this_fmode) { + /* This is an f-string. Concatenate and decref it. */ + Py_ssize_t ofs = 0; + int result = FstringParser_ConcatFstring(&state, s, &ofs, 0, c, n); + Py_DECREF(s); + if (result < 0) + goto error; + } else { + /* This is a regular string. Concatenate it. */ + if (FstringParser_ConcatAndDel(&state, s) < 0) + goto error; } } - return v; + if (bytesmode) { + /* Just return the bytes object and we're done. */ + if (PyArena_AddPyObject(c->c_arena, bytes_str) < 0) + goto error; + return Bytes(bytes_str, LINENO(n), n->n_col_offset, c->c_arena); + } + + /* We're not a bytes string, bytes_str should never have been set. */ + assert(bytes_str == NULL); + + return FstringParser_Finish(&state, c, n); - onError: - Py_XDECREF(v); +error: + Py_XDECREF(bytes_str); + FstringParser_Dealloc(&state); return NULL; } diff --git a/Python/compile.c b/Python/compile.c index a6884ec..3a49ece 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -731,6 +731,7 @@ compiler_set_qualname(struct compiler *c) return 1; } + /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -3209,6 +3210,117 @@ compiler_call(struct compiler *c, expr_ty e) e->v.Call.keywords); } +static int +compiler_joined_str(struct compiler *c, expr_ty e) +{ + /* Concatenate parts of a string using ''.join(parts). There are + probably better ways of doing this. + + This is used for constructs like "'x=' f'{42}'", which have to + be evaluated at compile time. */ + + static PyObject *empty_string; + static PyObject *join_string; + + if (!empty_string) { + empty_string = PyUnicode_FromString(""); + if (!empty_string) + return 0; + } + if (!join_string) { + join_string = PyUnicode_FromString("join"); + if (!join_string) + return 0; + } + + ADDOP_O(c, LOAD_CONST, empty_string, consts); + ADDOP_NAME(c, LOAD_ATTR, join_string, names); + VISIT_SEQ(c, expr, e->v.JoinedStr.values); + ADDOP_I(c, BUILD_LIST, asdl_seq_LEN(e->v.JoinedStr.values)); + ADDOP_I(c, CALL_FUNCTION, 1); + return 1; +} + +/* Note that this code uses the builtin functions format(), str(), + repr(), and ascii(). You can break this code, or make it do odd + things, by redefining those functions. */ +static int +compiler_formatted_value(struct compiler *c, expr_ty e) +{ + PyObject *conversion_name = NULL; + + static PyObject *format_string; + static PyObject *str_string; + static PyObject *repr_string; + static PyObject *ascii_string; + + if (!format_string) { + format_string = PyUnicode_InternFromString("format"); + if (!format_string) + return 0; + } + + if (!str_string) { + str_string = PyUnicode_InternFromString("str"); + if (!str_string) + return 0; + } + + if (!repr_string) { + repr_string = PyUnicode_InternFromString("repr"); + if (!repr_string) + return 0; + } + if (!ascii_string) { + ascii_string = PyUnicode_InternFromString("ascii"); + if (!ascii_string) + return 0; + } + + ADDOP_NAME(c, LOAD_GLOBAL, format_string, names); + + /* If needed, convert via str, repr, or ascii. */ + if (e->v.FormattedValue.conversion != -1) { + switch (e->v.FormattedValue.conversion) { + case 's': + conversion_name = str_string; + break; + case 'r': + conversion_name = repr_string; + break; + case 'a': + conversion_name = ascii_string; + break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; + } + ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names); + } + + /* Evaluate the value. */ + VISIT(c, expr, e->v.FormattedValue.value); + + /* If needed, convert via str, repr, or ascii. */ + if (conversion_name) { + /* Call the function we previously pushed. */ + ADDOP_I(c, CALL_FUNCTION, 1); + } + + /* If we have a format spec, use format(value, format_spec). Otherwise, + use the single argument form. */ + if (e->v.FormattedValue.format_spec) { + VISIT(c, expr, e->v.FormattedValue.format_spec); + ADDOP_I(c, CALL_FUNCTION, 2); + } else { + /* No format spec specified, call format(value). */ + ADDOP_I(c, CALL_FUNCTION, 1); + } + + return 1; +} + /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, @@ -3878,6 +3990,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e) case Str_kind: ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); break; + case JoinedStr_kind: + return compiler_joined_str(c, e); + case FormattedValue_kind: + return compiler_formatted_value(c, e); case Bytes_kind: ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); break; @@ -4784,4 +4900,3 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, { return PyAST_CompileEx(mod, filename, flags, -1, arena); } - diff --git a/Python/symtable.c b/Python/symtable.c index 64910d8..8431d51 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1439,6 +1439,14 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Call.args); VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords); break; + case FormattedValue_kind: + VISIT(st, expr, e->v.FormattedValue.value); + if (e->v.FormattedValue.format_spec) + VISIT(st, expr, e->v.FormattedValue.format_spec); + break; + case JoinedStr_kind: + VISIT_SEQ(st, expr, e->v.JoinedStr.values); + break; case Num_kind: case Str_kind: case Bytes_kind: -- cgit v0.12 From edef3ebafa3238b97d3bfe9913a904f19a1c195e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sat, 19 Sep 2015 15:49:57 -0400 Subject: Temporary hack for issue #25180: exclude test_fstring.py from the unparse round-tripping, while I figure out how to properly fix it. --- Lib/test/test_tools/test_unparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 976a6c5..920fcbd 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -264,6 +264,8 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): + if n == 'test_fstring.py': + continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) -- cgit v0.12 From 608adf9c827a14795c21a14aef98b90a94eac287 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Sun, 20 Sep 2015 15:09:15 -0400 Subject: Issue 25180: Fix Tools/parser/unparse.py for f-strings. Patch by Martin Panter. --- Lib/test/test_tools/test_unparse.py | 11 +++++++++-- Tools/parser/unparse.py | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tools/test_unparse.py b/Lib/test/test_tools/test_unparse.py index 920fcbd..4b47916 100644 --- a/Lib/test/test_tools/test_unparse.py +++ b/Lib/test/test_tools/test_unparse.py @@ -134,6 +134,15 @@ class ASTTestCase(unittest.TestCase): class UnparseTestCase(ASTTestCase): # Tests for specific bugs found in earlier versions of unparse + def test_fstrings(self): + # See issue 25180 + self.check_roundtrip(r"""f'{f"{0}"*3}'""") + self.check_roundtrip(r"""f'{f"{y}"*3}'""") + self.check_roundtrip(r"""f'{f"{\'x\'}"*3}'""") + + self.check_roundtrip(r'''f"{r'x' f'{\"s\"}'}"''') + self.check_roundtrip(r'''f"{r'x'rf'{\"s\"}'}"''') + def test_del_statement(self): self.check_roundtrip("del x, y, z") @@ -264,8 +273,6 @@ class DirectoryTestCase(ASTTestCase): for d in self.test_directories: test_dir = os.path.join(basepath, d) for n in os.listdir(test_dir): - if n == 'test_fstring.py': - continue if n.endswith('.py') and not n.startswith('bad'): names.append(os.path.join(test_dir, n)) diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py index c828577..9972797 100644 --- a/Tools/parser/unparse.py +++ b/Tools/parser/unparse.py @@ -322,6 +322,45 @@ class Unparser: def _Str(self, tree): self.write(repr(tree.s)) + def _JoinedStr(self, t): + self.write("f") + string = io.StringIO() + self._fstring_JoinedStr(t, string.write) + self.write(repr(string.getvalue())) + + def _FormattedValue(self, t): + self.write("f") + string = io.StringIO() + self._fstring_FormattedValue(t, string.write) + self.write(repr(string.getvalue())) + + def _fstring_JoinedStr(self, t, write): + for value in t.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, write) + + def _fstring_Str(self, t, write): + value = t.s.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_FormattedValue(self, t, write): + write("{") + expr = io.StringIO() + Unparser(t.value, expr) + expr = expr.getvalue().rstrip("\n") + if expr.startswith("{"): + write(" ") # Separate pair of opening brackets as "{ {" + write(expr) + if t.conversion != -1: + conversion = chr(t.conversion) + assert conversion in "sra" + write(f"!{conversion}") + if t.format_spec: + write(":") + meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) + meth(t.format_spec, write) + write("}") + def _Name(self, t): self.write(t.id) -- cgit v0.12 From 8e5bc9899522c9550e343039c207044f2c2f7c5f Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 20 Sep 2015 23:32:08 -0400 Subject: Add NEWS items for Idle to 3.6.0a1 section. --- Misc/NEWS | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index a5c5884..d16e18d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -78,6 +78,24 @@ Library - Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo functions. +IDLE +---- + +- Issue #25199: Idle: add synchronization comments for future maintainers. + +- Issue #16893: Replace help.txt with idle.html for Idle doc display. + The new idlelib/idle.html is copied from Doc/build/html/idle.html. + It looks better than help.txt and will better document Idle as released. + The tkinter html viewer that works for this file was written by Rose Roseman. + The now unused EditorWindow.HelpDialog class and helt.txt file are deprecated. + +- Issue #24199: Deprecate unused idlelib.idlever with possible removal in 3.6. + +- Issue #24782: In Idle extension config dialog, replace tabs with sorted list. + Patch by Mark Roseman. + +- Issue #24790: Remove extraneous code (which also create 2 & 3 conflicts). + Documentation ------------- -- cgit v0.12 From 5567f89c6f51a30034e07d6a4a678caf5b4bbb37 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 21 Sep 2015 13:36:09 -0400 Subject: Issue #24779: Remove unused rawmode parameter to unicode_decode. --- Python/ast.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 735424b..4b2dca4 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3938,7 +3938,7 @@ decode_utf8(struct compiling *c, const char **sPtr, const char *end) } static PyObject * -decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, const char *encoding) +decode_unicode(struct compiling *c, const char *s, size_t len, const char *encoding) { PyObject *v, *u; char *buf; @@ -3994,10 +3994,7 @@ decode_unicode(struct compiling *c, const char *s, size_t len, int rawmode, cons len = p - buf; s = buf; } - if (rawmode) - v = PyUnicode_DecodeRawUnicodeEscape(s, len, NULL); - else - v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); Py_XDECREF(u); return v; } @@ -4896,7 +4893,7 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) } } if (!*bytesmode && !rawmode) { - return decode_unicode(c, s, len, rawmode, c->c_encoding); + return decode_unicode(c, s, len, c->c_encoding); } if (*bytesmode) { /* Disallow non-ascii characters (but not escapes) */ -- cgit v0.12 From 1d59fee294d0bca792dacca4d5cc7dd21fedec95 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:29:43 +0200 Subject: Merge 3.5 (Issue #23630, fix test_asyncio) --- Lib/test/test_asyncio/test_events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 3488101..9801d22 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -764,6 +764,7 @@ class EventLoopTestsMixin: for host in hosts] self.loop.getaddrinfo = getaddrinfo_task self.loop._start_serving = mock.Mock() + self.loop._stop_serving = mock.Mock() f = self.loop.create_server(lambda: MyProto(self.loop), hosts, 80) server = self.loop.run_until_complete(f) self.addCleanup(server.close) -- cgit v0.12 From 4552ced91673b5138460e3d697b4bea8f4aab93b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:37:15 +0200 Subject: Issue #25207, #14626: Fix ICC compiler warnings in posixmodule.c Replace "#if XXX" with #ifdef XXX". --- Modules/posixmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6654fbb..e5b4792 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4609,7 +4609,7 @@ utime_fd(utime_t *ut, int fd) #define UTIME_HAVE_NOFOLLOW_SYMLINKS \ (defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES)) -#if UTIME_HAVE_NOFOLLOW_SYMLINKS +#ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS static int utime_nofollow_symlinks(utime_t *ut, char *path) @@ -4771,7 +4771,7 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, utime.now = 1; } -#if !UTIME_HAVE_NOFOLLOW_SYMLINKS +#if !defined(UTIME_HAVE_NOFOLLOW_SYMLINKS) if (follow_symlinks_specified("utime", follow_symlinks)) goto exit; #endif @@ -4825,7 +4825,7 @@ os_utime_impl(PyModuleDef *module, path_t *path, PyObject *times, #else /* MS_WINDOWS */ Py_BEGIN_ALLOW_THREADS -#if UTIME_HAVE_NOFOLLOW_SYMLINKS +#ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS if ((!follow_symlinks) && (dir_fd == DEFAULT_DIR_FD)) result = utime_nofollow_symlinks(&utime, path->narrow); else -- cgit v0.12 From ba4529593877f2016215149912496e030782a57e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 22:40:28 +0200 Subject: ssue #25207: fix ICC compiler warning in msvcrtmodule.c --- PC/msvcrtmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 52d4100..d73ce79 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -541,7 +541,6 @@ PyInit_msvcrt(void) #endif /* constants for the crt versions */ - (void)st; #ifdef _VC_ASSEMBLY_PUBLICKEYTOKEN st = PyModule_AddStringConstant(m, "VC_ASSEMBLY_PUBLICKEYTOKEN", _VC_ASSEMBLY_PUBLICKEYTOKEN); @@ -567,6 +566,8 @@ PyInit_msvcrt(void) st = PyModule_AddObject(m, "CRT_ASSEMBLY_VERSION", version); if (st < 0) return NULL; #endif + /* make compiler warning quiet if st is unused */ + (void)st; return m; } -- cgit v0.12 From f96418de05fc8710f9dc1e3a19878f8e744956f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Sep 2015 23:06:27 +0200 Subject: Issue #24870: Optimize the ASCII decoder for error handlers: surrogateescape, ignore and replace. Initial patch written by Naoki Inada. The decoder is now up to 60 times as fast for these error handlers. Add also unit tests for the ASCII decoder. --- Doc/whatsnew/3.6.rst | 3 ++- Lib/test/test_codecs.py | 32 ++++++++++++++++++++++++ Objects/unicodeobject.c | 66 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 48ff38a..8a2b5d3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -106,7 +106,8 @@ operator Optimizations ============= -* None yet. +* The ASCII decoder is now up to 60 times as fast for error handlers: + ``surrogateescape``, ``ignore`` and ``replace``. Build and C API Changes diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a4a6f95..e0e3119 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -27,6 +27,7 @@ def coding_checker(self, coder): self.assertEqual(coder(input), (expect, len(input))) return check + class Queue(object): """ queue: write bytes at one end, read bytes from the other end @@ -47,6 +48,7 @@ class Queue(object): self._buffer = self._buffer[size:] return s + class MixInCheckStateHandling: def check_state_handling_decode(self, encoding, u, s): for i in range(len(s)+1): @@ -80,6 +82,7 @@ class MixInCheckStateHandling: part2 = d.encode(u[i:], True) self.assertEqual(s, part1+part2) + class ReadTest(MixInCheckStateHandling): def check_partial(self, input, partialresults): # get a StreamReader for the encoding and feed the bytestring version @@ -383,6 +386,7 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), before + backslashreplace + after) + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" if sys.byteorder == 'little': @@ -478,6 +482,7 @@ class UTF32Test(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) + class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" @@ -523,6 +528,7 @@ class UTF32LETest(ReadTest, unittest.TestCase): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) + class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" @@ -797,6 +803,7 @@ class UTF8Test(ReadTest, unittest.TestCase): with self.assertRaises(UnicodeDecodeError): b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + @unittest.skipUnless(sys.platform == 'win32', 'cp65001 is a Windows-only codec') class CP65001Test(ReadTest, unittest.TestCase): @@ -1136,6 +1143,7 @@ class EscapeDecodeTest(unittest.TestCase): self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8)) self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8)) + class RecodingTest(unittest.TestCase): def test_recoding(self): f = io.BytesIO() @@ -1255,6 +1263,7 @@ for i in punycode_testcases: if len(i)!=2: print(repr(i)) + class PunycodeTest(unittest.TestCase): def test_encode(self): for uni, puny in punycode_testcases: @@ -1274,6 +1283,7 @@ class PunycodeTest(unittest.TestCase): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + class UnicodeInternalTest(unittest.TestCase): @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): @@ -1528,6 +1538,7 @@ class NameprepTest(unittest.TestCase): except Exception as e: raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e))) + class IDNACodecTest(unittest.TestCase): def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") @@ -1614,6 +1625,7 @@ class IDNACodecTest(unittest.TestCase): self.assertRaises(Exception, b"python.org".decode, "idna", errors) + class CodecsModuleTest(unittest.TestCase): def test_decode(self): @@ -1722,6 +1734,7 @@ class CodecsModuleTest(unittest.TestCase): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + class StreamReaderTest(unittest.TestCase): def setUp(self): @@ -1732,6 +1745,7 @@ class StreamReaderTest(unittest.TestCase): f = self.reader(self.stream) self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00']) + class EncodedFileTest(unittest.TestCase): def test_basic(self): @@ -1862,6 +1876,7 @@ broken_unicode_with_stateful = [ "unicode_internal" ] + class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_basics(self): s = "abc123" # all codecs should be able to encode these @@ -2024,6 +2039,7 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): self.check_state_handling_decode(encoding, u, u.encode(encoding)) self.check_state_handling_encode(encoding, u, u.encode(encoding)) + class CharmapTest(unittest.TestCase): def test_decode_with_string_map(self): self.assertEqual( @@ -2274,6 +2290,7 @@ class WithStmtTest(unittest.TestCase): info.streamwriter, 'strict') as srw: self.assertEqual(srw.read(), "\xfc") + class TypesTest(unittest.TestCase): def test_decode_unicode(self): # Most decoders don't accept unicode input @@ -2564,6 +2581,7 @@ else: bytes_transform_encodings.append("bz2_codec") transform_aliases["bz2_codec"] = ["bz2"] + class TransformCodecTest(unittest.TestCase): def test_basics(self): @@ -3041,5 +3059,19 @@ class CodePageTest(unittest.TestCase): self.assertEqual(decoded, ('abc', 3)) +class ASCIITest(unittest.TestCase): + def test_decode(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode('ascii', error_handler), + expected) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0709789..b8840e1 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6644,6 +6644,28 @@ PyUnicode_AsLatin1String(PyObject *unicode) /* --- 7-bit ASCII Codec -------------------------------------------------- */ +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_OTHER +} _Py_error_handler; + +static _Py_error_handler +get_error_handler(const char *errors) +{ + if (errors == NULL) + return _Py_ERROR_OTHER; + if (strcmp(errors, "surrogateescape") == 0) + return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "ignore") == 0) + return _Py_ERROR_IGNORE; + if (strcmp(errors, "replace") == 0) + return _Py_ERROR_REPLACE; + return _Py_ERROR_OTHER; +} + PyObject * PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, @@ -6657,8 +6679,9 @@ PyUnicode_DecodeASCII(const char *s, Py_ssize_t endinpos; Py_ssize_t outpos; const char *e; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) _Py_RETURN_UNICODE_EMPTY(); @@ -6687,12 +6710,45 @@ PyUnicode_DecodeASCII(const char *s, PyUnicode_WRITE(kind, data, writer.pos, c); writer.pos++; ++s; + continue; } - else { + + /* byte outsize range 0x00..0x7f: call the error handler */ + + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + case _Py_ERROR_SURROGATEESCAPE: + /* Fast-path: the error handler only writes one character, + but we must switch to UCS2 at the first write */ + if (kind < PyUnicode_2BYTE_KIND) { + if (_PyUnicodeWriter_Prepare(&writer, size - writer.pos, + 0xffff) < 0) + return NULL; + kind = writer.kind; + data = writer.data; + } + + if (error_handler == _Py_ERROR_REPLACE) + PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); + else + PyUnicode_WRITE(kind, data, writer.pos, c + 0xdc00); + writer.pos++; + ++s; + break; + + case _Py_ERROR_IGNORE: + ++s; + break; + + default: startinpos = s-starts; endinpos = startinpos + 1; if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, + errors, &error_handler_obj, "ascii", "ordinal not in range(128)", &starts, &e, &startinpos, &endinpos, &exc, &s, &writer)) @@ -6701,13 +6757,13 @@ PyUnicode_DecodeASCII(const char *s, data = writer.data; } } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: _PyUnicodeWriter_Dealloc(&writer); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } -- cgit v0.12 From 5014920cb72768bc54924e55e7004e79fcad94f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 00:26:54 +0200 Subject: Issue #24870: Reuse the new _Py_error_handler enum Factorize code with the new get_error_handler() function. Add some empty lines for readability. --- Objects/unicodeobject.c | 164 +++++++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 87 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index b8840e1..f5f2d48 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -293,6 +293,34 @@ static unsigned char ascii_linebreak[] = { #include "clinic/unicodeobject.c.h" +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_STRICT, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_XMLCHARREFREPLACE, + _Py_ERROR_OTHER +} _Py_error_handler; + +static _Py_error_handler +get_error_handler(const char *errors) +{ + if (errors == NULL) + return _Py_ERROR_STRICT; + if (strcmp(errors, "strict") == 0) + return _Py_ERROR_STRICT; + if (strcmp(errors, "surrogateescape") == 0) + return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "ignore") == 0) + return _Py_ERROR_IGNORE; + if (strcmp(errors, "replace") == 0) + return _Py_ERROR_REPLACE; + if (strcmp(errors, "xmlcharrefreplace") == 0) + return _Py_ERROR_XMLCHARREFREPLACE; + return _Py_ERROR_OTHER; +} + /* The max unicode value is always 0x10FFFF while using the PEP-393 API. This function is kept for backward compatibility with the old API. */ Py_UNICODE @@ -3163,24 +3191,22 @@ wcstombs_errorpos(const wchar_t *wstr) static int locale_error_handler(const char *errors, int *surrogateescape) { - if (errors == NULL) { - *surrogateescape = 0; - return 0; - } - - if (strcmp(errors, "strict") == 0) { + _Py_error_handler error_handler = get_error_handler(errors); + switch (error_handler) + { + case _Py_ERROR_STRICT: *surrogateescape = 0; return 0; - } - if (strcmp(errors, "surrogateescape") == 0) { + case _Py_ERROR_SURROGATEESCAPE: *surrogateescape = 1; return 0; + default: + PyErr_Format(PyExc_ValueError, + "only 'strict' and 'surrogateescape' error handlers " + "are supported, not '%s'", + errors); + return -1; } - PyErr_Format(PyExc_ValueError, - "only 'strict' and 'surrogateescape' error handlers " - "are supported, not '%s'", - errors); - return -1; } PyObject * @@ -6403,11 +6429,9 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; - /* the following variable is used for caching string comparisons - * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */ - int known_errorHandler = -1; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (PyUnicode_READY(unicode) == -1) return NULL; @@ -6441,32 +6465,28 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t collstart = pos; Py_ssize_t collend = pos; /* find all unecodable characters */ + while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) ++collend; + /* cache callback name lookup (if not done yet, i.e. it's the first error) */ - if (known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - known_errorHandler = 4; - else - known_errorHandler = 0; - } - switch (known_errorHandler) { - case 1: /* strict */ + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(&exc, encoding, unicode, collstart, collend, reason); goto onError; - case 2: /* replace */ + + case _Py_ERROR_REPLACE: while (collstart++ < collend) - *str++ = '?'; /* fall through */ - case 3: /* ignore */ + *str++ = '?'; + /* fall through */ + case _Py_ERROR_IGNORE: pos = collend; break; - case 4: /* xmlcharrefreplace */ + + case _Py_ERROR_XMLCHARREFREPLACE: respos = str - PyBytes_AS_STRING(res); requiredsize = respos; /* determine replacement size */ @@ -6510,8 +6530,9 @@ unicode_encode_ucs1(PyObject *unicode, } pos = collend; break; + default: - repunicode = unicode_encode_call_errorhandler(errors, &errorHandler, + repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, encoding, reason, unicode, &exc, collstart, collend, &newpos); if (repunicode == NULL || (PyUnicode_Check(repunicode) && @@ -6587,7 +6608,7 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; } - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return res; @@ -6597,7 +6618,7 @@ unicode_encode_ucs1(PyObject *unicode, onError: Py_XDECREF(res); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; } @@ -6644,28 +6665,6 @@ PyUnicode_AsLatin1String(PyObject *unicode) /* --- 7-bit ASCII Codec -------------------------------------------------- */ -typedef enum { - _Py_ERROR_UNKNOWN=0, - _Py_ERROR_SURROGATEESCAPE, - _Py_ERROR_REPLACE, - _Py_ERROR_IGNORE, - _Py_ERROR_OTHER -} _Py_error_handler; - -static _Py_error_handler -get_error_handler(const char *errors) -{ - if (errors == NULL) - return _Py_ERROR_OTHER; - if (strcmp(errors, "surrogateescape") == 0) - return _Py_ERROR_SURROGATEESCAPE; - if (strcmp(errors, "ignore") == 0) - return _Py_ERROR_IGNORE; - if (strcmp(errors, "replace") == 0) - return _Py_ERROR_REPLACE; - return _Py_ERROR_OTHER; -} - PyObject * PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, @@ -8129,7 +8128,7 @@ static int charmap_encoding_error( PyObject *unicode, Py_ssize_t *inpos, PyObject *mapping, PyObject **exceptionObject, - int *known_errorHandler, PyObject **errorHandler, const char *errors, + _Py_error_handler *error_handler, PyObject **error_handler_obj, const char *errors, PyObject **res, Py_ssize_t *respos) { PyObject *repunicode = NULL; /* initialize to prevent gcc warning */ @@ -8176,23 +8175,15 @@ charmap_encoding_error( } /* cache callback name lookup * (if not done yet, i.e. it's the first error) */ - if (*known_errorHandler==-1) { - if ((errors==NULL) || (!strcmp(errors, "strict"))) - *known_errorHandler = 1; - else if (!strcmp(errors, "replace")) - *known_errorHandler = 2; - else if (!strcmp(errors, "ignore")) - *known_errorHandler = 3; - else if (!strcmp(errors, "xmlcharrefreplace")) - *known_errorHandler = 4; - else - *known_errorHandler = 0; - } - switch (*known_errorHandler) { - case 1: /* strict */ + if (*error_handler == _Py_ERROR_UNKNOWN) + *error_handler = get_error_handler(errors); + + switch (*error_handler) { + case _Py_ERROR_STRICT: raise_encode_exception(exceptionObject, encoding, unicode, collstartpos, collendpos, reason); return -1; - case 2: /* replace */ + + case _Py_ERROR_REPLACE: for (collpos = collstartpos; collpos Date: Tue, 22 Sep 2015 00:58:32 +0200 Subject: Issue #24870: Add _PyUnicodeWriter_PrepareKind() macro Add a macro which ensures that the writer has at least the requested kind. --- Include/unicodeobject.h | 17 +++++++++++++++++ Objects/unicodeobject.c | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 33e8f19..d0e0142 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -942,6 +942,23 @@ PyAPI_FUNC(int) _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar); +/* Prepare the buffer to have at least the kind KIND. + For example, kind=PyUnicode_2BYTE_KIND ensures that the writer will + support characters in range U+000-U+FFFF. + + Return 0 on success, raise an exception and return -1 on error. */ +#define _PyUnicodeWriter_PrepareKind(WRITER, KIND) \ + (assert((KIND) != PyUnicode_WCHAR_KIND), \ + (KIND) <= (WRITER)->kind \ + ? 0 \ + : _PyUnicodeWriter_PrepareKindInternal((WRITER), (KIND))) + +/* Don't call this function directly, use the _PyUnicodeWriter_PrepareKind() + macro instead. */ +PyAPI_FUNC(int) +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + enum PyUnicode_Kind kind); + /* Append a Unicode character. Return 0 on success, raise an exception and return -1 on error. */ PyAPI_FUNC(int) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index f5f2d48..7c079e0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6722,14 +6722,11 @@ PyUnicode_DecodeASCII(const char *s, case _Py_ERROR_REPLACE: case _Py_ERROR_SURROGATEESCAPE: /* Fast-path: the error handler only writes one character, - but we must switch to UCS2 at the first write */ - if (kind < PyUnicode_2BYTE_KIND) { - if (_PyUnicodeWriter_Prepare(&writer, size - writer.pos, - 0xffff) < 0) - return NULL; - kind = writer.kind; - data = writer.data; - } + but we may switch to UCS2 at the first write */ + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + kind = writer.kind; + data = writer.data; if (error_handler == _Py_ERROR_REPLACE) PyUnicode_WRITE(kind, data, writer.pos, 0xfffd); @@ -13309,7 +13306,8 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t newlen; PyObject *newbuffer; - assert(length > 0); + /* ensure that the _PyUnicodeWriter_Prepare macro was used */ + assert(maxchar > writer->maxchar || length > 0); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); @@ -13375,6 +13373,28 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, #undef OVERALLOCATE_FACTOR } +int +_PyUnicodeWriter_PrepareKindInternal(_PyUnicodeWriter *writer, + enum PyUnicode_Kind kind) +{ + Py_UCS4 maxchar; + + /* ensure that the _PyUnicodeWriter_PrepareKind macro was used */ + assert(writer->kind < kind); + + switch (kind) + { + case PyUnicode_1BYTE_KIND: maxchar = 0xff; break; + case PyUnicode_2BYTE_KIND: maxchar = 0xffff; break; + case PyUnicode_4BYTE_KIND: maxchar = 0x10ffff; break; + default: + assert(0 && "invalid kind"); + return -1; + } + + return _PyUnicodeWriter_PrepareInternal(writer, 0, maxchar); +} + Py_LOCAL_INLINE(int) _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) { -- cgit v0.12 From 6174474bea9fe6f5f12f05a16004eabb817ce721 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 01:01:17 +0200 Subject: _PyUnicodeWriter_PrepareInternal(): make the assertion more strict --- Objects/unicodeobject.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7c079e0..d0b285a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13307,7 +13307,8 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, PyObject *newbuffer; /* ensure that the _PyUnicodeWriter_Prepare macro was used */ - assert(maxchar > writer->maxchar || length > 0); + assert((maxchar > writer->maxchar && length >= 0) + || length > 0); if (length > PY_SSIZE_T_MAX - writer->pos) { PyErr_NoMemory(); -- cgit v0.12 From 5ebae876281828c17f139ec063dae43a39fd7741 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 01:29:33 +0200 Subject: Issue #25207, #14626: Fix my commit. It doesn't work to use #define XXX defined(YYY)" and then "#ifdef XXX" to check YYY. --- Modules/posixmodule.c | 6 +++--- Objects/unicodeobject.c | 52 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e5b4792..467ce2c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4605,9 +4605,9 @@ utime_fd(utime_t *ut, int fd) #define PATH_UTIME_HAVE_FD 0 #endif - -#define UTIME_HAVE_NOFOLLOW_SYMLINKS \ - (defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES)) +#if defined(HAVE_UTIMENSAT) || defined(HAVE_LUTIMES) +# define UTIME_HAVE_NOFOLLOW_SYMLINKS +#endif #ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d0b285a..63a627f 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4709,8 +4709,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4773,24 +4774,57 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + /* undecodable byte: call the error handler */ + + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + case _Py_ERROR_SURROGATEESCAPE: + { + unsigned char ch = (unsigned char)*s; + + /* Fast-path: the error handler only writes one character, + but we may switch to UCS2 at the first write */ + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + kind = writer.kind; + + if (error_handler == _Py_ERROR_REPLACE) + PyUnicode_WRITE(kind, writer.data, writer.pos, 0xfffd); + else + PyUnicode_WRITE(kind, writer.data, writer.pos, ch + 0xdc00); + writer.pos++; + ++s; + break; + } + + case _Py_ERROR_IGNORE: + s++; + break; + + default: + if (unicode_decode_call_errorhandler_writer( + errors, &error_handler_obj, + "utf-8", errmsg, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; + } } End: if (consumed) *consumed = s - starts; - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); _PyUnicodeWriter_Dealloc(&writer); return NULL; -- cgit v0.12 From d3d2b2c50c8e85e1d3f9429964e080d0ea5f8634 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 21 Sep 2015 23:41:56 -0700 Subject: Minor consistency improvements to negative value comparisons. --- Modules/_collectionsmodule.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3e2c69f..7dbf755 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -209,7 +209,7 @@ deque_pop(dequeobject *deque, PyObject *unused) Py_SIZE(deque)--; deque->state++; - if (deque->rightindex == -1) { + if (deque->rightindex < 0) { if (Py_SIZE(deque)) { prevblock = deque->rightblock->leftlink; assert(deque->leftblock != deque->rightblock); @@ -715,7 +715,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) *(dest++) = *(src++); } while (--m); } - if (rightindex == -1) { + if (rightindex < 0) { assert(leftblock != rightblock); assert(b == NULL); b = rightblock; @@ -827,7 +827,7 @@ deque_reverse(dequeobject *deque, PyObject *unused) /* Step backwards with the right block/index pair */ rightindex--; - if (rightindex == -1) { + if (rightindex < 0) { rightblock = rightblock->leftlink; rightindex = BLOCKLEN - 1; } @@ -1234,7 +1234,7 @@ deque_copy(PyObject *deque) Py_DECREF(new_deque); return NULL; } - if (old_deque->maxlen == -1) + if (old_deque->maxlen < 0) return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); else return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", @@ -1258,12 +1258,12 @@ deque_reduce(dequeobject *deque) return NULL; } if (dict == NULL) { - if (deque->maxlen == -1) + if (deque->maxlen < 0) result = Py_BuildValue("O(O)", Py_TYPE(deque), aslist); else result = Py_BuildValue("O(On)", Py_TYPE(deque), aslist, deque->maxlen); } else { - if (deque->maxlen == -1) + if (deque->maxlen < 0) result = Py_BuildValue("O(OO)O", Py_TYPE(deque), aslist, Py_None, dict); else result = Py_BuildValue("O(On)O", Py_TYPE(deque), aslist, deque->maxlen, dict); @@ -1354,7 +1354,7 @@ deque_richcompare(PyObject *v, PyObject *w, int op) } Py_DECREF(x); Py_DECREF(y); - if (b == -1) + if (b < 0) goto done; } /* We reached the end of one deque or both */ @@ -1437,7 +1437,7 @@ deque_bool(dequeobject *deque) static PyObject * deque_get_maxlen(dequeobject *deque) { - if (deque->maxlen == -1) + if (deque->maxlen < 0) Py_RETURN_NONE; return PyLong_FromSsize_t(deque->maxlen); } @@ -1778,7 +1778,7 @@ dequereviter_next(dequeiterobject *it) item = it->b->data[it->index]; it->index--; it->counter--; - if (it->index == -1 && it->counter > 0) { + if (it->index < 0 && it->counter > 0) { CHECK_NOT_END(it->b->leftlink); it->b = it->b->leftlink; it->index = BLOCKLEN - 1; -- cgit v0.12 From 7a237230d1a98a369de7e3c6581a42d31465430a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 22 Sep 2015 01:20:36 -0700 Subject: Eliminate unnecessary variable --- Modules/_collectionsmodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7dbf755..4eb31bf 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -804,10 +804,9 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_ssize_t leftindex = deque->leftindex; Py_ssize_t rightindex = deque->rightindex; Py_ssize_t n = Py_SIZE(deque) >> 1; - Py_ssize_t i; PyObject *tmp; - for (i=0 ; i 0) { /* Validate that pointers haven't met in the middle */ assert(leftblock != rightblock || leftindex < rightindex); CHECK_NOT_END(leftblock); -- cgit v0.12 From 54385b206d5a91e17eaf7d54cadb049847fa1c8a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Sep 2015 10:46:52 +0200 Subject: Issue #24870: revert unwanted change Sorry, I pushed the patch on the UTF-8 decoder by mistake :-( --- Objects/unicodeobject.c | 52 +++++++++---------------------------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 63a627f..d0b285a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4709,9 +4709,8 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *error_handler_obj = NULL; + PyObject *errorHandler = NULL; PyObject *exc = NULL; - _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4774,57 +4773,24 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - /* undecodable byte: call the error handler */ - - if (error_handler == _Py_ERROR_UNKNOWN) - error_handler = get_error_handler(errors); - - switch (error_handler) - { - case _Py_ERROR_REPLACE: - case _Py_ERROR_SURROGATEESCAPE: - { - unsigned char ch = (unsigned char)*s; - - /* Fast-path: the error handler only writes one character, - but we may switch to UCS2 at the first write */ - if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) - goto onError; - kind = writer.kind; - - if (error_handler == _Py_ERROR_REPLACE) - PyUnicode_WRITE(kind, writer.data, writer.pos, 0xfffd); - else - PyUnicode_WRITE(kind, writer.data, writer.pos, ch + 0xdc00); - writer.pos++; - ++s; - break; - } - - case _Py_ERROR_IGNORE: - s++; - break; - - default: - if (unicode_decode_call_errorhandler_writer( - errors, &error_handler_obj, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; - } + if (unicode_decode_call_errorhandler_writer( + errors, &errorHandler, + "utf-8", errmsg, + &starts, &end, &startinpos, &endinpos, &exc, &s, + &writer)) + goto onError; } End: if (consumed) *consumed = s - starts; - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); return _PyUnicodeWriter_Finish(&writer); onError: - Py_XDECREF(error_handler_obj); + Py_XDECREF(errorHandler); Py_XDECREF(exc); _PyUnicodeWriter_Dealloc(&writer); return NULL; -- cgit v0.12 From 3a1a8d04244cb5d98e8bbeaff131a705f84b8521 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 23 Sep 2015 02:42:02 -0700 Subject: Eliminate unnecessary variables --- Modules/_collectionsmodule.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4eb31bf..be6c90c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -843,13 +843,12 @@ deque_count(dequeobject *deque, PyObject *v) block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; Py_ssize_t n = Py_SIZE(deque); - Py_ssize_t i; Py_ssize_t count = 0; size_t start_state = deque->state; PyObject *item; int cmp; - for (i=0 ; idata[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); @@ -883,12 +882,11 @@ deque_contains(dequeobject *deque, PyObject *v) block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; Py_ssize_t n = Py_SIZE(deque); - Py_ssize_t i; size_t start_state = deque->state; PyObject *item; int cmp; - for (i=0 ; idata[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); -- cgit v0.12 From 1d44c41b0cc183603b31ac943ef452456bef089f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 07:49:00 -0400 Subject: Move f-string compilation of the expression earlier, before the conversion character and format_spec are checked. This allows for error messages that more closely match what a user would expect. --- Lib/test/test_fstring.py | 9 +++++++ Python/ast.c | 66 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index a6ff9cf..2a1a3cd 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -287,6 +287,15 @@ f'{a * x()}'""" "f' { } '", r"f'{\n}'", r"f'{\n \n}'", + + # Catch the empty expression before the + # invalid conversion. + "f'{!x}'", + "f'{ !xr}'", + "f'{!x:}'", + "f'{!x:a}'", + "f'{ !xr:}'", + "f'{ !xr:a}'", ]) def test_parens_in_expressions(self): diff --git a/Python/ast.c b/Python/ast.c index 4b2dca4..83e4f00 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4007,13 +4007,14 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod expression. This is to allow strings with embedded newlines, for example. */ static expr_ty -fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, - Py_ssize_t expr_end, PyArena *arena) +fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, + Py_ssize_t expr_end, PyArena *arena) { PyCompilerFlags cf; mod_ty mod; char *utf_expr; Py_ssize_t i; + Py_UCS4 end_ch = -1; int all_whitespace; PyObject *sub = NULL; @@ -4023,6 +4024,16 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, assert(str); + assert(expr_start >= 0 && expr_start < PyUnicode_GET_LENGTH(str)); + assert(expr_end >= 0 && expr_end < PyUnicode_GET_LENGTH(str)); + assert(expr_end >= expr_start); + + /* There has to be at least on character on each side of the + expression inside this str. This will have been caught before + we're called. */ + assert(expr_start >= 1); + assert(expr_end <= PyUnicode_GET_LENGTH(str)-1); + /* If the substring is all whitespace, it's an error. We need to catch this here, and not when we call PyParser_ASTFromString, because turning the expression '' in to '()' would go from @@ -4049,10 +4060,17 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, string directly. */ if (expr_start-1 == 0 && expr_end+1 == PyUnicode_GET_LENGTH(str)) { - /* No need to actually remember these characters, because we - know they must be braces. */ + /* If str is well formed, then the first and last chars must + be '{' and '}', respectively. But, if there's a syntax + error, for example f'{3!', then the last char won't be a + closing brace. So, remember the last character we read in + order for us to restore it. */ + end_ch = PyUnicode_ReadChar(str, expr_end-expr_start+1); + assert(end_ch != (Py_UCS4)-1); + + /* In all cases, however, start_ch must be '{'. */ assert(PyUnicode_ReadChar(str, 0) == '{'); - assert(PyUnicode_ReadChar(str, expr_end-expr_start+1) == '}'); + sub = str; } else { /* Create a substring object. It must be a new object, with @@ -4064,21 +4082,23 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, decref_sub = 1; /* Remember to deallocate it on error. */ } + /* Put () around the expression. */ if (PyUnicode_WriteChar(sub, 0, '(') < 0 || PyUnicode_WriteChar(sub, expr_end-expr_start+1, ')') < 0) goto error; - cf.cf_flags = PyCF_ONLY_AST; - /* No need to free the memory returned here: it's managed by the string. */ utf_expr = PyUnicode_AsUTF8(sub); if (!utf_expr) goto error; + + cf.cf_flags = PyCF_ONLY_AST; mod = PyParser_ASTFromString(utf_expr, "", Py_eval_input, &cf, arena); if (!mod) goto error; + if (sub != str) /* Clear instead of decref in case we ever modify this code to change the error handling: this is safest because the XDECREF won't try @@ -4089,9 +4109,10 @@ fstring_expression_compile(PyObject *str, Py_ssize_t expr_start, Py_CLEAR(sub); else { assert(!decref_sub); + assert(end_ch != (Py_UCS4)-1); /* Restore str, which we earlier modified directly. */ if (PyUnicode_WriteChar(str, 0, '{') < 0 || - PyUnicode_WriteChar(str, expr_end-expr_start+1, '}') < 0) + PyUnicode_WriteChar(str, expr_end-expr_start+1, end_ch) < 0) goto error; } return mod->v.Expression.body; @@ -4100,6 +4121,18 @@ error: /* Only decref sub if it was the result of a call to SubString. */ if (decref_sub) Py_XDECREF(sub); + + if (end_ch != (Py_UCS4)-1) { + /* We only get here if we modified str. Make sure that's the + case: str will be equal to sub. */ + if (str == sub) { + /* Don't check the error, because we've already set the + error state (that's why we're in 'error', after + all). */ + PyUnicode_WriteChar(str, 0, '{'); + PyUnicode_WriteChar(str, expr_end-expr_start+1, end_ch); + } + } return NULL; } @@ -4331,9 +4364,18 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, return -1; } - /* Check for a conversion char, if present. */ if (*ofs >= PyUnicode_GET_LENGTH(str)) goto unexpected_end_of_string; + + /* Compile the expression as soon as possible, so we show errors + related to the expression before errors related to the + conversion or format_spec. */ + simple_expression = fstring_compile_expr(str, expr_start, expr_end, + c->c_arena); + if (!simple_expression) + return -1; + + /* Check for a conversion char, if present. */ if (PyUnicode_READ(kind, data, *ofs) == '!') { *ofs += 1; if (*ofs >= PyUnicode_GET_LENGTH(str)) @@ -4374,12 +4416,6 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, assert(PyUnicode_READ(kind, data, *ofs) == '}'); *ofs += 1; - /* Compile the expression. */ - simple_expression = fstring_expression_compile(str, expr_start, expr_end, - c->c_arena); - if (!simple_expression) - return -1; - /* And now create the FormattedValue node that represents this entire expression with the conversion and format spec. */ *expression = FormattedValue(simple_expression, (int)conversion, -- cgit v0.12 From 548c4d31784048d64eb11ab13652ca0d4abb6d39 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 08:00:01 -0400 Subject: Added more f-string test for empty expressions. --- Lib/test/test_fstring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 2a1a3cd..10c5849 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -296,6 +296,9 @@ f'{a * x()}'""" "f'{!x:a}'", "f'{ !xr:}'", "f'{ !xr:a}'", + + "f'{!}'", + "f'{:}'", ]) def test_parens_in_expressions(self): -- cgit v0.12 From b2080f65544093c8f07b4ecc43fe035aeb08c8f0 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 23 Sep 2015 10:24:43 -0400 Subject: f-strings: More tests for empty expressions along with missing closing braces. --- Lib/test/test_fstring.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 10c5849..d6f781c 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -299,6 +299,13 @@ f'{a * x()}'""" "f'{!}'", "f'{:}'", + + # We find the empty expression before the + # missing closing brace. + "f'{!'", + "f'{!s:'", + "f'{:'", + "f'{:x'", ]) def test_parens_in_expressions(self): -- cgit v0.12 From 7b92abf0e972158d20585cc7ef5c72aec56811cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:04:18 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000..554aeed --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000..9464996 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66b..51e9e18 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f..cf4b84f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 0cca00b67edc10bbbf6eae00cb79756fa365033e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2015 23:16:47 +0200 Subject: Issue #25220: Backed out changeset eaf9a99b6bb8 --- Lib/test/libregrtest/__init__.py | 1 - Lib/test/libregrtest/cmdline.py | 340 --------------------------------------- Lib/test/regrtest.py | 332 +++++++++++++++++++++++++++++++++++++- Lib/test/test_regrtest.py | 4 +- 4 files changed, 332 insertions(+), 345 deletions(-) delete mode 100644 Lib/test/libregrtest/__init__.py delete mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py deleted file mode 100644 index 554aeed..0000000 --- a/Lib/test/libregrtest/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py deleted file mode 100644 index 9464996..0000000 --- a/Lib/test/libregrtest/cmdline.py +++ /dev/null @@ -1,340 +0,0 @@ -import argparse -import faulthandler -import os - -from test import support - -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - - -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18..b49e66b 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,6 +6,119 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + # We import importlib *ASAP* in order to test #15386 import importlib @@ -40,8 +153,6 @@ try: except ImportError: multiprocessing = None -from test.libregrtest import _parse_args - # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -87,6 +198,9 @@ CHILD_ERROR = -5 # error in a child process from test import support +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -96,6 +210,220 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns + + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84f..a398a4f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import regrtest, support class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(libregrtest.RESOURCE_NAMES) + expected = list(regrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 2b0d646b751e6e5bdb466c05ffd2824dd46c7d60 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 23 Sep 2015 19:15:44 -0700 Subject: Replace an unpredictable branch with a simple addition. --- Modules/_collectionsmodule.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index be6c90c..8d753c3 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -852,10 +852,9 @@ deque_count(dequeobject *deque, PyObject *v) CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); - if (cmp > 0) - count++; - else if (cmp < 0) + if (cmp < 0) return NULL; + count += cmp; if (start_state != deque->state) { PyErr_SetString(PyExc_RuntimeError, -- cgit v0.12 From 1e5fcc3dea4263a5c01ecc4cd9b5d755fab5ee6a Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Thu, 24 Sep 2015 08:52:04 -0400 Subject: Fixed error creation if the problem is an empty expression in an f-string: use ast_error instead of PyErr_SetString. --- Python/ast.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 83e4f00..7eab3c0 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4008,7 +4008,8 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod example. */ static expr_ty fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, - Py_ssize_t expr_end, PyArena *arena) + Py_ssize_t expr_end, struct compiling *c, const node *n) + { PyCompilerFlags cf; mod_ty mod; @@ -4048,8 +4049,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, } } if (all_whitespace) { - PyErr_SetString(PyExc_SyntaxError, "f-string: empty expression " - "not allowed"); + ast_error(c, n, "f-string: empty expression not allowed"); goto error; } @@ -4095,7 +4095,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, cf.cf_flags = PyCF_ONLY_AST; mod = PyParser_ASTFromString(utf_expr, "", - Py_eval_input, &cf, arena); + Py_eval_input, &cf, c->c_arena); if (!mod) goto error; @@ -4370,8 +4370,7 @@ fstring_find_expr(PyObject *str, Py_ssize_t *ofs, int recurse_lvl, /* Compile the expression as soon as possible, so we show errors related to the expression before errors related to the conversion or format_spec. */ - simple_expression = fstring_compile_expr(str, expr_start, expr_end, - c->c_arena); + simple_expression = fstring_compile_expr(str, expr_start, expr_end, c, n); if (!simple_expression) return -1; -- cgit v0.12 From 0030cd52dacdd95d2017a0947d661feb737449af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Sep 2015 14:45:00 +0200 Subject: Issue #25227: Cleanup unicode_encode_ucs1() error handler * Change limit type from unsigned int to Py_UCS4, to use the same type than the "ch" variable (an Unicode character). * Reuse ch variable for _Py_ERROR_XMLCHARREFREPLACE * Add some newlines for readability --- Objects/unicodeobject.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index d0b285a..da2aac7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6415,7 +6415,7 @@ unicode_encode_call_errorhandler(const char *errors, static PyObject * unicode_encode_ucs1(PyObject *unicode, const char *errors, - unsigned int limit) + const Py_UCS4 limit) { /* input state */ Py_ssize_t pos=0, size; @@ -6449,12 +6449,12 @@ unicode_encode_ucs1(PyObject *unicode, ressize = size; while (pos < size) { - Py_UCS4 c = PyUnicode_READ(kind, data, pos); + Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* can we encode this? */ - if (c0; ++i, ++str) { - c = PyUnicode_READ_CHAR(repunicode, i); - if (c >= limit) { + ch = PyUnicode_READ_CHAR(repunicode, i); + if (ch >= limit) { raise_encode_exception(&exc, encoding, unicode, pos, pos+1, reason); Py_DECREF(repunicode); goto onError; } - *str = (char)c; + *str = (char)ch; } pos = newpos; Py_DECREF(repunicode); -- cgit v0.12 From 1dae0c68dd98fca70de121a61d28493a3e92a3ac Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 25 Sep 2015 13:05:13 -0700 Subject: Issue #25186: Remove duplicated function from importlib._bootstrap_external --- Lib/importlib/_bootstrap_external.py | 48 +- Python/importlib_external.h | 4977 +++++++++++++++++----------------- 2 files changed, 2502 insertions(+), 2523 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 3508ce9..abde6d9 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -360,14 +360,6 @@ def _calc_mode(path): return mode -def _verbose_message(message, *args, verbosity=1): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -437,15 +429,15 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): raw_size = data[8:12] if magic != MAGIC_NUMBER: message = 'bad magic number in {!r}: {!r}'.format(name, magic) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) elif len(raw_timestamp) != 4: message = 'reached EOF while reading timestamp in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) elif len(raw_size) != 4: message = 'reached EOF while reading size of source in {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise EOFError(message) if source_stats is not None: try: @@ -455,7 +447,7 @@ def _validate_bytecode_header(data, source_stats=None, name=None, path=None): else: if _r_long(raw_timestamp) != source_mtime: message = 'bytecode is stale for {!r}'.format(name) - _verbose_message(message) + _bootstrap._verbose_message(message) raise ImportError(message, **exc_details) try: source_size = source_stats['size'] & 0xFFFFFFFF @@ -472,7 +464,7 @@ def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): """Compile bytecode as returned by _validate_bytecode_header().""" code = marshal.loads(data) if isinstance(code, _code_type): - _verbose_message('code object from {!r}', bytecode_path) + _bootstrap._verbose_message('code object from {!r}', bytecode_path) if source_path is not None: _imp._fix_co_filename(code, source_path) return code @@ -755,21 +747,21 @@ class SourceLoader(_LoaderBasics): except (ImportError, EOFError): pass else: - _verbose_message('{} matches {}', bytecode_path, - source_path) + _bootstrap._verbose_message('{} matches {}', bytecode_path, + source_path) return _compile_bytecode(bytes_data, name=fullname, bytecode_path=bytecode_path, source_path=source_path) source_bytes = self.get_data(source_path) code_object = self.source_to_code(source_bytes, source_path) - _verbose_message('code object from {}', source_path) + _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): data = _code_to_bytecode(code_object, source_mtime, len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) - _verbose_message('wrote {!r}', bytecode_path) + _bootstrap._verbose_message('wrote {!r}', bytecode_path) except NotImplementedError: pass return code_object @@ -849,14 +841,16 @@ class SourceFileLoader(FileLoader, SourceLoader): except OSError as exc: # Could be a permission error, read-only filesystem: just forget # about writing the data. - _verbose_message('could not create {!r}: {!r}', parent, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', + parent, exc) return try: _write_atomic(path, data, _mode) - _verbose_message('created {!r}', path) + _bootstrap._verbose_message('created {!r}', path) except OSError as exc: # Same as above: just don't write the bytecode. - _verbose_message('could not create {!r}: {!r}', path, exc) + _bootstrap._verbose_message('could not create {!r}: {!r}', path, + exc) class SourcelessFileLoader(FileLoader, _LoaderBasics): @@ -901,14 +895,14 @@ class ExtensionFileLoader(FileLoader, _LoaderBasics): """Create an unitialized extension module""" module = _bootstrap._call_with_frames_removed( _imp.create_dynamic, spec) - _verbose_message('extension module {!r} loaded from {!r}', + _bootstrap._verbose_message('extension module {!r} loaded from {!r}', spec.name, self.path) return module def exec_module(self, module): """Initialize an extension module""" _bootstrap._call_with_frames_removed(_imp.exec_dynamic, module) - _verbose_message('extension module {!r} executed from {!r}', + _bootstrap._verbose_message('extension module {!r} executed from {!r}', self.name, self.path) def is_package(self, fullname): @@ -1023,7 +1017,8 @@ class _NamespaceLoader: """ # The import system never calls this method. - _verbose_message('namespace module loaded with path {!r}', self._path) + _bootstrap._verbose_message('namespace module loaded with path {!r}', + self._path) return _bootstrap._load_module_shim(self, fullname) @@ -1243,12 +1238,13 @@ class FileFinder: # Check for a file w/ a proper suffix exists. for suffix, loader_class in self._loaders: full_path = _path_join(self.path, tail_module + suffix) - _verbose_message('trying {}'.format(full_path), verbosity=2) + _bootstrap._verbose_message('trying {}', full_path, verbosity=2) if cache_module + suffix in cache: if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, None, target) + return self._get_spec(loader_class, fullname, full_path, + None, target) if is_namespace: - _verbose_message('possible namespace for {}'.format(base_path)) + _bootstrap._verbose_message('possible namespace for {}', base_path) spec = _bootstrap.ModuleSpec(fullname, None) spec.submodule_search_locations = [base_path] return spec diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 48d6be5..f4f5ac4 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1,8 +1,8 @@ /* Auto-generated by Programs/_freeze_importlib.c */ const unsigned char _Py_M__importlib_external[] = { 99,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0, - 0,64,0,0,0,115,228,2,0,0,100,0,0,90,0,0, - 100,96,0,90,1,0,100,4,0,100,5,0,132,0,0,90, + 0,64,0,0,0,115,210,2,0,0,100,0,0,90,0,0, + 100,92,0,90,1,0,100,4,0,100,5,0,132,0,0,90, 2,0,100,6,0,100,7,0,132,0,0,90,3,0,100,8, 0,100,9,0,132,0,0,90,4,0,100,10,0,100,11,0, 132,0,0,90,5,0,100,12,0,100,13,0,132,0,0,90, @@ -20,563 +20,541 @@ const unsigned char _Py_M__importlib_external[] = { 1,1,90,26,0,100,37,0,100,38,0,132,0,0,90,27, 0,100,39,0,100,40,0,132,0,0,90,28,0,100,41,0, 100,42,0,132,0,0,90,29,0,100,43,0,100,44,0,132, - 0,0,90,30,0,100,45,0,100,46,0,100,47,0,100,48, - 0,132,0,1,90,31,0,100,49,0,100,50,0,132,0,0, - 90,32,0,100,51,0,100,52,0,132,0,0,90,33,0,100, - 33,0,100,33,0,100,33,0,100,53,0,100,54,0,132,3, - 0,90,34,0,100,33,0,100,33,0,100,33,0,100,55,0, - 100,56,0,132,3,0,90,35,0,100,57,0,100,57,0,100, - 58,0,100,59,0,132,2,0,90,36,0,100,60,0,100,61, - 0,132,0,0,90,37,0,101,38,0,131,0,0,90,39,0, - 100,33,0,100,62,0,100,33,0,100,63,0,101,39,0,100, - 64,0,100,65,0,132,1,2,90,40,0,71,100,66,0,100, - 67,0,132,0,0,100,67,0,131,2,0,90,41,0,71,100, - 68,0,100,69,0,132,0,0,100,69,0,131,2,0,90,42, - 0,71,100,70,0,100,71,0,132,0,0,100,71,0,101,42, - 0,131,3,0,90,43,0,71,100,72,0,100,73,0,132,0, - 0,100,73,0,131,2,0,90,44,0,71,100,74,0,100,75, - 0,132,0,0,100,75,0,101,44,0,101,43,0,131,4,0, - 90,45,0,71,100,76,0,100,77,0,132,0,0,100,77,0, - 101,44,0,101,42,0,131,4,0,90,46,0,103,0,0,90, - 47,0,71,100,78,0,100,79,0,132,0,0,100,79,0,101, - 44,0,101,42,0,131,4,0,90,48,0,71,100,80,0,100, - 81,0,132,0,0,100,81,0,131,2,0,90,49,0,71,100, - 82,0,100,83,0,132,0,0,100,83,0,131,2,0,90,50, - 0,71,100,84,0,100,85,0,132,0,0,100,85,0,131,2, - 0,90,51,0,71,100,86,0,100,87,0,132,0,0,100,87, - 0,131,2,0,90,52,0,100,33,0,100,88,0,100,89,0, - 132,1,0,90,53,0,100,90,0,100,91,0,132,0,0,90, - 54,0,100,92,0,100,93,0,132,0,0,90,55,0,100,94, - 0,100,95,0,132,0,0,90,56,0,100,33,0,83,41,97, - 97,94,1,0,0,67,111,114,101,32,105,109,112,108,101,109, - 101,110,116,97,116,105,111,110,32,111,102,32,112,97,116,104, - 45,98,97,115,101,100,32,105,109,112,111,114,116,46,10,10, - 84,104,105,115,32,109,111,100,117,108,101,32,105,115,32,78, - 79,84,32,109,101,97,110,116,32,116,111,32,98,101,32,100, - 105,114,101,99,116,108,121,32,105,109,112,111,114,116,101,100, - 33,32,73,116,32,104,97,115,32,98,101,101,110,32,100,101, - 115,105,103,110,101,100,32,115,117,99,104,10,116,104,97,116, - 32,105,116,32,99,97,110,32,98,101,32,98,111,111,116,115, - 116,114,97,112,112,101,100,32,105,110,116,111,32,80,121,116, - 104,111,110,32,97,115,32,116,104,101,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, - 111,114,116,46,32,65,115,10,115,117,99,104,32,105,116,32, - 114,101,113,117,105,114,101,115,32,116,104,101,32,105,110,106, - 101,99,116,105,111,110,32,111,102,32,115,112,101,99,105,102, - 105,99,32,109,111,100,117,108,101,115,32,97,110,100,32,97, - 116,116,114,105,98,117,116,101,115,32,105,110,32,111,114,100, - 101,114,32,116,111,10,119,111,114,107,46,32,79,110,101,32, - 115,104,111,117,108,100,32,117,115,101,32,105,109,112,111,114, - 116,108,105,98,32,97,115,32,116,104,101,32,112,117,98,108, - 105,99,45,102,97,99,105,110,103,32,118,101,114,115,105,111, - 110,32,111,102,32,116,104,105,115,32,109,111,100,117,108,101, - 46,10,10,218,3,119,105,110,218,6,99,121,103,119,105,110, - 218,6,100,97,114,119,105,110,99,0,0,0,0,0,0,0, - 0,1,0,0,0,2,0,0,0,67,0,0,0,115,49,0, - 0,0,116,0,0,106,1,0,106,2,0,116,3,0,131,1, - 0,114,33,0,100,1,0,100,2,0,132,0,0,125,0,0, - 110,12,0,100,3,0,100,2,0,132,0,0,125,0,0,124, - 0,0,83,41,4,78,99,0,0,0,0,0,0,0,0,0, - 0,0,0,2,0,0,0,83,0,0,0,115,13,0,0,0, - 100,1,0,116,0,0,106,1,0,107,6,0,83,41,2,122, - 53,84,114,117,101,32,105,102,32,102,105,108,101,110,97,109, - 101,115,32,109,117,115,116,32,98,101,32,99,104,101,99,107, - 101,100,32,99,97,115,101,45,105,110,115,101,110,115,105,116, - 105,118,101,108,121,46,115,12,0,0,0,80,89,84,72,79, - 78,67,65,83,69,79,75,41,2,218,3,95,111,115,90,7, - 101,110,118,105,114,111,110,169,0,114,4,0,0,0,114,4, - 0,0,0,250,38,60,102,114,111,122,101,110,32,105,109,112, - 111,114,116,108,105,98,46,95,98,111,111,116,115,116,114,97, - 112,95,101,120,116,101,114,110,97,108,62,218,11,95,114,101, - 108,97,120,95,99,97,115,101,30,0,0,0,115,2,0,0, - 0,0,2,122,37,95,109,97,107,101,95,114,101,108,97,120, - 95,99,97,115,101,46,60,108,111,99,97,108,115,62,46,95, - 114,101,108,97,120,95,99,97,115,101,99,0,0,0,0,0, - 0,0,0,0,0,0,0,1,0,0,0,83,0,0,0,115, - 4,0,0,0,100,1,0,83,41,2,122,53,84,114,117,101, - 32,105,102,32,102,105,108,101,110,97,109,101,115,32,109,117, - 115,116,32,98,101,32,99,104,101,99,107,101,100,32,99,97, - 115,101,45,105,110,115,101,110,115,105,116,105,118,101,108,121, - 46,70,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,6,0,0,0, - 34,0,0,0,115,2,0,0,0,0,2,41,4,218,3,115, - 121,115,218,8,112,108,97,116,102,111,114,109,218,10,115,116, - 97,114,116,115,119,105,116,104,218,27,95,67,65,83,69,95, - 73,78,83,69,78,83,73,84,73,86,69,95,80,76,65,84, - 70,79,82,77,83,41,1,114,6,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,16,95,109,97, - 107,101,95,114,101,108,97,120,95,99,97,115,101,28,0,0, - 0,115,8,0,0,0,0,1,18,1,15,4,12,3,114,11, + 0,0,90,30,0,100,45,0,100,46,0,132,0,0,90,31, + 0,100,47,0,100,48,0,132,0,0,90,32,0,100,33,0, + 100,33,0,100,33,0,100,49,0,100,50,0,132,3,0,90, + 33,0,100,33,0,100,33,0,100,33,0,100,51,0,100,52, + 0,132,3,0,90,34,0,100,53,0,100,53,0,100,54,0, + 100,55,0,132,2,0,90,35,0,100,56,0,100,57,0,132, + 0,0,90,36,0,101,37,0,131,0,0,90,38,0,100,33, + 0,100,58,0,100,33,0,100,59,0,101,38,0,100,60,0, + 100,61,0,132,1,2,90,39,0,71,100,62,0,100,63,0, + 132,0,0,100,63,0,131,2,0,90,40,0,71,100,64,0, + 100,65,0,132,0,0,100,65,0,131,2,0,90,41,0,71, + 100,66,0,100,67,0,132,0,0,100,67,0,101,41,0,131, + 3,0,90,42,0,71,100,68,0,100,69,0,132,0,0,100, + 69,0,131,2,0,90,43,0,71,100,70,0,100,71,0,132, + 0,0,100,71,0,101,43,0,101,42,0,131,4,0,90,44, + 0,71,100,72,0,100,73,0,132,0,0,100,73,0,101,43, + 0,101,41,0,131,4,0,90,45,0,103,0,0,90,46,0, + 71,100,74,0,100,75,0,132,0,0,100,75,0,101,43,0, + 101,41,0,131,4,0,90,47,0,71,100,76,0,100,77,0, + 132,0,0,100,77,0,131,2,0,90,48,0,71,100,78,0, + 100,79,0,132,0,0,100,79,0,131,2,0,90,49,0,71, + 100,80,0,100,81,0,132,0,0,100,81,0,131,2,0,90, + 50,0,71,100,82,0,100,83,0,132,0,0,100,83,0,131, + 2,0,90,51,0,100,33,0,100,84,0,100,85,0,132,1, + 0,90,52,0,100,86,0,100,87,0,132,0,0,90,53,0, + 100,88,0,100,89,0,132,0,0,90,54,0,100,90,0,100, + 91,0,132,0,0,90,55,0,100,33,0,83,41,93,97,94, + 1,0,0,67,111,114,101,32,105,109,112,108,101,109,101,110, + 116,97,116,105,111,110,32,111,102,32,112,97,116,104,45,98, + 97,115,101,100,32,105,109,112,111,114,116,46,10,10,84,104, + 105,115,32,109,111,100,117,108,101,32,105,115,32,78,79,84, + 32,109,101,97,110,116,32,116,111,32,98,101,32,100,105,114, + 101,99,116,108,121,32,105,109,112,111,114,116,101,100,33,32, + 73,116,32,104,97,115,32,98,101,101,110,32,100,101,115,105, + 103,110,101,100,32,115,117,99,104,10,116,104,97,116,32,105, + 116,32,99,97,110,32,98,101,32,98,111,111,116,115,116,114, + 97,112,112,101,100,32,105,110,116,111,32,80,121,116,104,111, + 110,32,97,115,32,116,104,101,32,105,109,112,108,101,109,101, + 110,116,97,116,105,111,110,32,111,102,32,105,109,112,111,114, + 116,46,32,65,115,10,115,117,99,104,32,105,116,32,114,101, + 113,117,105,114,101,115,32,116,104,101,32,105,110,106,101,99, + 116,105,111,110,32,111,102,32,115,112,101,99,105,102,105,99, + 32,109,111,100,117,108,101,115,32,97,110,100,32,97,116,116, + 114,105,98,117,116,101,115,32,105,110,32,111,114,100,101,114, + 32,116,111,10,119,111,114,107,46,32,79,110,101,32,115,104, + 111,117,108,100,32,117,115,101,32,105,109,112,111,114,116,108, + 105,98,32,97,115,32,116,104,101,32,112,117,98,108,105,99, + 45,102,97,99,105,110,103,32,118,101,114,115,105,111,110,32, + 111,102,32,116,104,105,115,32,109,111,100,117,108,101,46,10, + 10,218,3,119,105,110,218,6,99,121,103,119,105,110,218,6, + 100,97,114,119,105,110,99,0,0,0,0,0,0,0,0,1, + 0,0,0,2,0,0,0,67,0,0,0,115,49,0,0,0, + 116,0,0,106,1,0,106,2,0,116,3,0,131,1,0,114, + 33,0,100,1,0,100,2,0,132,0,0,125,0,0,110,12, + 0,100,3,0,100,2,0,132,0,0,125,0,0,124,0,0, + 83,41,4,78,99,0,0,0,0,0,0,0,0,0,0,0, + 0,2,0,0,0,83,0,0,0,115,13,0,0,0,100,1, + 0,116,0,0,106,1,0,107,6,0,83,41,2,122,53,84, + 114,117,101,32,105,102,32,102,105,108,101,110,97,109,101,115, + 32,109,117,115,116,32,98,101,32,99,104,101,99,107,101,100, + 32,99,97,115,101,45,105,110,115,101,110,115,105,116,105,118, + 101,108,121,46,115,12,0,0,0,80,89,84,72,79,78,67, + 65,83,69,79,75,41,2,218,3,95,111,115,90,7,101,110, + 118,105,114,111,110,169,0,114,4,0,0,0,114,4,0,0, + 0,250,38,60,102,114,111,122,101,110,32,105,109,112,111,114, + 116,108,105,98,46,95,98,111,111,116,115,116,114,97,112,95, + 101,120,116,101,114,110,97,108,62,218,11,95,114,101,108,97, + 120,95,99,97,115,101,30,0,0,0,115,2,0,0,0,0, + 2,122,37,95,109,97,107,101,95,114,101,108,97,120,95,99, + 97,115,101,46,60,108,111,99,97,108,115,62,46,95,114,101, + 108,97,120,95,99,97,115,101,99,0,0,0,0,0,0,0, + 0,0,0,0,0,1,0,0,0,83,0,0,0,115,4,0, + 0,0,100,1,0,83,41,2,122,53,84,114,117,101,32,105, + 102,32,102,105,108,101,110,97,109,101,115,32,109,117,115,116, + 32,98,101,32,99,104,101,99,107,101,100,32,99,97,115,101, + 45,105,110,115,101,110,115,105,116,105,118,101,108,121,46,70, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,6,0,0,0,34,0, + 0,0,115,2,0,0,0,0,2,41,4,218,3,115,121,115, + 218,8,112,108,97,116,102,111,114,109,218,10,115,116,97,114, + 116,115,119,105,116,104,218,27,95,67,65,83,69,95,73,78, + 83,69,78,83,73,84,73,86,69,95,80,76,65,84,70,79, + 82,77,83,41,1,114,6,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,16,95,109,97,107,101, + 95,114,101,108,97,120,95,99,97,115,101,28,0,0,0,115, + 8,0,0,0,0,1,18,1,15,4,12,3,114,11,0,0, + 0,99,1,0,0,0,0,0,0,0,1,0,0,0,3,0, + 0,0,67,0,0,0,115,26,0,0,0,116,0,0,124,0, + 0,131,1,0,100,1,0,64,106,1,0,100,2,0,100,3, + 0,131,2,0,83,41,4,122,42,67,111,110,118,101,114,116, + 32,97,32,51,50,45,98,105,116,32,105,110,116,101,103,101, + 114,32,116,111,32,108,105,116,116,108,101,45,101,110,100,105, + 97,110,46,108,3,0,0,0,255,127,255,127,3,0,233,4, + 0,0,0,218,6,108,105,116,116,108,101,41,2,218,3,105, + 110,116,218,8,116,111,95,98,121,116,101,115,41,1,218,1, + 120,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,7,95,119,95,108,111,110,103,40,0,0,0,115,2,0, + 0,0,0,2,114,17,0,0,0,99,1,0,0,0,0,0, + 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,16, + 0,0,0,116,0,0,106,1,0,124,0,0,100,1,0,131, + 2,0,83,41,2,122,47,67,111,110,118,101,114,116,32,52, + 32,98,121,116,101,115,32,105,110,32,108,105,116,116,108,101, + 45,101,110,100,105,97,110,32,116,111,32,97,110,32,105,110, + 116,101,103,101,114,46,114,13,0,0,0,41,2,114,14,0, + 0,0,218,10,102,114,111,109,95,98,121,116,101,115,41,1, + 90,9,105,110,116,95,98,121,116,101,115,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,7,95,114,95,108, + 111,110,103,45,0,0,0,115,2,0,0,0,0,2,114,19, + 0,0,0,99,0,0,0,0,0,0,0,0,1,0,0,0, + 3,0,0,0,71,0,0,0,115,26,0,0,0,116,0,0, + 106,1,0,100,1,0,100,2,0,132,0,0,124,0,0,68, + 131,1,0,131,1,0,83,41,3,122,31,82,101,112,108,97, + 99,101,109,101,110,116,32,102,111,114,32,111,115,46,112,97, + 116,104,46,106,111,105,110,40,41,46,99,1,0,0,0,0, + 0,0,0,2,0,0,0,4,0,0,0,83,0,0,0,115, + 37,0,0,0,103,0,0,124,0,0,93,27,0,125,1,0, + 124,1,0,114,6,0,124,1,0,106,0,0,116,1,0,131, + 1,0,145,2,0,113,6,0,83,114,4,0,0,0,41,2, + 218,6,114,115,116,114,105,112,218,15,112,97,116,104,95,115, + 101,112,97,114,97,116,111,114,115,41,2,218,2,46,48,218, + 4,112,97,114,116,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,250,10,60,108,105,115,116,99,111,109,112,62, + 52,0,0,0,115,2,0,0,0,9,1,122,30,95,112,97, + 116,104,95,106,111,105,110,46,60,108,111,99,97,108,115,62, + 46,60,108,105,115,116,99,111,109,112,62,41,2,218,8,112, + 97,116,104,95,115,101,112,218,4,106,111,105,110,41,1,218, + 10,112,97,116,104,95,112,97,114,116,115,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,10,95,112,97,116, + 104,95,106,111,105,110,50,0,0,0,115,4,0,0,0,0, + 2,15,1,114,28,0,0,0,99,1,0,0,0,0,0,0, + 0,5,0,0,0,5,0,0,0,67,0,0,0,115,134,0, + 0,0,116,0,0,116,1,0,131,1,0,100,1,0,107,2, + 0,114,52,0,124,0,0,106,2,0,116,3,0,131,1,0, + 92,3,0,125,1,0,125,2,0,125,3,0,124,1,0,124, + 3,0,102,2,0,83,120,69,0,116,4,0,124,0,0,131, + 1,0,68,93,55,0,125,4,0,124,4,0,116,1,0,107, + 6,0,114,65,0,124,0,0,106,5,0,124,4,0,100,2, + 0,100,1,0,131,1,1,92,2,0,125,1,0,125,3,0, + 124,1,0,124,3,0,102,2,0,83,113,65,0,87,100,3, + 0,124,0,0,102,2,0,83,41,4,122,32,82,101,112,108, + 97,99,101,109,101,110,116,32,102,111,114,32,111,115,46,112, + 97,116,104,46,115,112,108,105,116,40,41,46,233,1,0,0, + 0,90,8,109,97,120,115,112,108,105,116,218,0,41,6,218, + 3,108,101,110,114,21,0,0,0,218,10,114,112,97,114,116, + 105,116,105,111,110,114,25,0,0,0,218,8,114,101,118,101, + 114,115,101,100,218,6,114,115,112,108,105,116,41,5,218,4, + 112,97,116,104,90,5,102,114,111,110,116,218,1,95,218,4, + 116,97,105,108,114,16,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,11,95,112,97,116,104,95, + 115,112,108,105,116,56,0,0,0,115,16,0,0,0,0,2, + 18,1,24,1,10,1,19,1,12,1,27,1,14,1,114,38, 0,0,0,99,1,0,0,0,0,0,0,0,1,0,0,0, - 3,0,0,0,67,0,0,0,115,26,0,0,0,116,0,0, - 124,0,0,131,1,0,100,1,0,64,106,1,0,100,2,0, - 100,3,0,131,2,0,83,41,4,122,42,67,111,110,118,101, - 114,116,32,97,32,51,50,45,98,105,116,32,105,110,116,101, - 103,101,114,32,116,111,32,108,105,116,116,108,101,45,101,110, - 100,105,97,110,46,108,3,0,0,0,255,127,255,127,3,0, - 233,4,0,0,0,218,6,108,105,116,116,108,101,41,2,218, - 3,105,110,116,218,8,116,111,95,98,121,116,101,115,41,1, - 218,1,120,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,119,95,108,111,110,103,40,0,0,0,115, - 2,0,0,0,0,2,114,17,0,0,0,99,1,0,0,0, - 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, - 115,16,0,0,0,116,0,0,106,1,0,124,0,0,100,1, - 0,131,2,0,83,41,2,122,47,67,111,110,118,101,114,116, - 32,52,32,98,121,116,101,115,32,105,110,32,108,105,116,116, - 108,101,45,101,110,100,105,97,110,32,116,111,32,97,110,32, - 105,110,116,101,103,101,114,46,114,13,0,0,0,41,2,114, - 14,0,0,0,218,10,102,114,111,109,95,98,121,116,101,115, - 41,1,90,9,105,110,116,95,98,121,116,101,115,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,7,95,114, - 95,108,111,110,103,45,0,0,0,115,2,0,0,0,0,2, - 114,19,0,0,0,99,0,0,0,0,0,0,0,0,1,0, - 0,0,3,0,0,0,71,0,0,0,115,26,0,0,0,116, - 0,0,106,1,0,100,1,0,100,2,0,132,0,0,124,0, - 0,68,131,1,0,131,1,0,83,41,3,122,31,82,101,112, + 2,0,0,0,67,0,0,0,115,13,0,0,0,116,0,0, + 106,1,0,124,0,0,131,1,0,83,41,1,122,126,83,116, + 97,116,32,116,104,101,32,112,97,116,104,46,10,10,32,32, + 32,32,77,97,100,101,32,97,32,115,101,112,97,114,97,116, + 101,32,102,117,110,99,116,105,111,110,32,116,111,32,109,97, + 107,101,32,105,116,32,101,97,115,105,101,114,32,116,111,32, + 111,118,101,114,114,105,100,101,32,105,110,32,101,120,112,101, + 114,105,109,101,110,116,115,10,32,32,32,32,40,101,46,103, + 46,32,99,97,99,104,101,32,115,116,97,116,32,114,101,115, + 117,108,116,115,41,46,10,10,32,32,32,32,41,2,114,3, + 0,0,0,90,4,115,116,97,116,41,1,114,35,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, + 10,95,112,97,116,104,95,115,116,97,116,68,0,0,0,115, + 2,0,0,0,0,7,114,39,0,0,0,99,2,0,0,0, + 0,0,0,0,3,0,0,0,11,0,0,0,67,0,0,0, + 115,58,0,0,0,121,16,0,116,0,0,124,0,0,131,1, + 0,125,2,0,87,110,22,0,4,116,1,0,107,10,0,114, + 40,0,1,1,1,100,1,0,83,89,110,1,0,88,124,2, + 0,106,2,0,100,2,0,64,124,1,0,107,2,0,83,41, + 3,122,49,84,101,115,116,32,119,104,101,116,104,101,114,32, + 116,104,101,32,112,97,116,104,32,105,115,32,116,104,101,32, + 115,112,101,99,105,102,105,101,100,32,109,111,100,101,32,116, + 121,112,101,46,70,105,0,240,0,0,41,3,114,39,0,0, + 0,218,7,79,83,69,114,114,111,114,218,7,115,116,95,109, + 111,100,101,41,3,114,35,0,0,0,218,4,109,111,100,101, + 90,9,115,116,97,116,95,105,110,102,111,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,18,95,112,97,116, + 104,95,105,115,95,109,111,100,101,95,116,121,112,101,78,0, + 0,0,115,10,0,0,0,0,2,3,1,16,1,13,1,9, + 1,114,43,0,0,0,99,1,0,0,0,0,0,0,0,1, + 0,0,0,3,0,0,0,67,0,0,0,115,13,0,0,0, + 116,0,0,124,0,0,100,1,0,131,2,0,83,41,2,122, + 31,82,101,112,108,97,99,101,109,101,110,116,32,102,111,114, + 32,111,115,46,112,97,116,104,46,105,115,102,105,108,101,46, + 105,0,128,0,0,41,1,114,43,0,0,0,41,1,114,35, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,12,95,112,97,116,104,95,105,115,102,105,108,101, + 87,0,0,0,115,2,0,0,0,0,2,114,44,0,0,0, + 99,1,0,0,0,0,0,0,0,1,0,0,0,3,0,0, + 0,67,0,0,0,115,31,0,0,0,124,0,0,115,18,0, + 116,0,0,106,1,0,131,0,0,125,0,0,116,2,0,124, + 0,0,100,1,0,131,2,0,83,41,2,122,30,82,101,112, 108,97,99,101,109,101,110,116,32,102,111,114,32,111,115,46, - 112,97,116,104,46,106,111,105,110,40,41,46,99,1,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,83,0,0, - 0,115,37,0,0,0,103,0,0,124,0,0,93,27,0,125, - 1,0,124,1,0,114,6,0,124,1,0,106,0,0,116,1, - 0,131,1,0,145,2,0,113,6,0,83,114,4,0,0,0, - 41,2,218,6,114,115,116,114,105,112,218,15,112,97,116,104, - 95,115,101,112,97,114,97,116,111,114,115,41,2,218,2,46, - 48,218,4,112,97,114,116,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,250,10,60,108,105,115,116,99,111,109, - 112,62,52,0,0,0,115,2,0,0,0,9,1,122,30,95, - 112,97,116,104,95,106,111,105,110,46,60,108,111,99,97,108, - 115,62,46,60,108,105,115,116,99,111,109,112,62,41,2,218, - 8,112,97,116,104,95,115,101,112,218,4,106,111,105,110,41, - 1,218,10,112,97,116,104,95,112,97,114,116,115,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,10,95,112, - 97,116,104,95,106,111,105,110,50,0,0,0,115,4,0,0, - 0,0,2,15,1,114,28,0,0,0,99,1,0,0,0,0, - 0,0,0,5,0,0,0,5,0,0,0,67,0,0,0,115, - 134,0,0,0,116,0,0,116,1,0,131,1,0,100,1,0, - 107,2,0,114,52,0,124,0,0,106,2,0,116,3,0,131, - 1,0,92,3,0,125,1,0,125,2,0,125,3,0,124,1, - 0,124,3,0,102,2,0,83,120,69,0,116,4,0,124,0, - 0,131,1,0,68,93,55,0,125,4,0,124,4,0,116,1, - 0,107,6,0,114,65,0,124,0,0,106,5,0,124,4,0, - 100,2,0,100,1,0,131,1,1,92,2,0,125,1,0,125, - 3,0,124,1,0,124,3,0,102,2,0,83,113,65,0,87, - 100,3,0,124,0,0,102,2,0,83,41,4,122,32,82,101, - 112,108,97,99,101,109,101,110,116,32,102,111,114,32,111,115, - 46,112,97,116,104,46,115,112,108,105,116,40,41,46,233,1, - 0,0,0,90,8,109,97,120,115,112,108,105,116,218,0,41, - 6,218,3,108,101,110,114,21,0,0,0,218,10,114,112,97, - 114,116,105,116,105,111,110,114,25,0,0,0,218,8,114,101, - 118,101,114,115,101,100,218,6,114,115,112,108,105,116,41,5, - 218,4,112,97,116,104,90,5,102,114,111,110,116,218,1,95, - 218,4,116,97,105,108,114,16,0,0,0,114,4,0,0,0, + 112,97,116,104,46,105,115,100,105,114,46,105,0,64,0,0, + 41,3,114,3,0,0,0,218,6,103,101,116,99,119,100,114, + 43,0,0,0,41,1,114,35,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,95,112,97,116, - 104,95,115,112,108,105,116,56,0,0,0,115,16,0,0,0, - 0,2,18,1,24,1,10,1,19,1,12,1,27,1,14,1, - 114,38,0,0,0,99,1,0,0,0,0,0,0,0,1,0, - 0,0,2,0,0,0,67,0,0,0,115,13,0,0,0,116, - 0,0,106,1,0,124,0,0,131,1,0,83,41,1,122,126, - 83,116,97,116,32,116,104,101,32,112,97,116,104,46,10,10, - 32,32,32,32,77,97,100,101,32,97,32,115,101,112,97,114, - 97,116,101,32,102,117,110,99,116,105,111,110,32,116,111,32, - 109,97,107,101,32,105,116,32,101,97,115,105,101,114,32,116, - 111,32,111,118,101,114,114,105,100,101,32,105,110,32,101,120, - 112,101,114,105,109,101,110,116,115,10,32,32,32,32,40,101, - 46,103,46,32,99,97,99,104,101,32,115,116,97,116,32,114, - 101,115,117,108,116,115,41,46,10,10,32,32,32,32,41,2, - 114,3,0,0,0,90,4,115,116,97,116,41,1,114,35,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,10,95,112,97,116,104,95,115,116,97,116,68,0,0, - 0,115,2,0,0,0,0,7,114,39,0,0,0,99,2,0, - 0,0,0,0,0,0,3,0,0,0,11,0,0,0,67,0, - 0,0,115,58,0,0,0,121,16,0,116,0,0,124,0,0, - 131,1,0,125,2,0,87,110,22,0,4,116,1,0,107,10, - 0,114,40,0,1,1,1,100,1,0,83,89,110,1,0,88, - 124,2,0,106,2,0,100,2,0,64,124,1,0,107,2,0, - 83,41,3,122,49,84,101,115,116,32,119,104,101,116,104,101, - 114,32,116,104,101,32,112,97,116,104,32,105,115,32,116,104, - 101,32,115,112,101,99,105,102,105,101,100,32,109,111,100,101, - 32,116,121,112,101,46,70,105,0,240,0,0,41,3,114,39, - 0,0,0,218,7,79,83,69,114,114,111,114,218,7,115,116, - 95,109,111,100,101,41,3,114,35,0,0,0,218,4,109,111, - 100,101,90,9,115,116,97,116,95,105,110,102,111,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,18,95,112, - 97,116,104,95,105,115,95,109,111,100,101,95,116,121,112,101, - 78,0,0,0,115,10,0,0,0,0,2,3,1,16,1,13, - 1,9,1,114,43,0,0,0,99,1,0,0,0,0,0,0, - 0,1,0,0,0,3,0,0,0,67,0,0,0,115,13,0, - 0,0,116,0,0,124,0,0,100,1,0,131,2,0,83,41, - 2,122,31,82,101,112,108,97,99,101,109,101,110,116,32,102, - 111,114,32,111,115,46,112,97,116,104,46,105,115,102,105,108, - 101,46,105,0,128,0,0,41,1,114,43,0,0,0,41,1, - 114,35,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,12,95,112,97,116,104,95,105,115,102,105, - 108,101,87,0,0,0,115,2,0,0,0,0,2,114,44,0, - 0,0,99,1,0,0,0,0,0,0,0,1,0,0,0,3, - 0,0,0,67,0,0,0,115,31,0,0,0,124,0,0,115, - 18,0,116,0,0,106,1,0,131,0,0,125,0,0,116,2, - 0,124,0,0,100,1,0,131,2,0,83,41,2,122,30,82, - 101,112,108,97,99,101,109,101,110,116,32,102,111,114,32,111, - 115,46,112,97,116,104,46,105,115,100,105,114,46,105,0,64, - 0,0,41,3,114,3,0,0,0,218,6,103,101,116,99,119, - 100,114,43,0,0,0,41,1,114,35,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,112, - 97,116,104,95,105,115,100,105,114,92,0,0,0,115,6,0, - 0,0,0,2,6,1,12,1,114,46,0,0,0,105,182,1, - 0,0,99,3,0,0,0,0,0,0,0,6,0,0,0,17, - 0,0,0,67,0,0,0,115,193,0,0,0,100,1,0,106, - 0,0,124,0,0,116,1,0,124,0,0,131,1,0,131,2, - 0,125,3,0,116,2,0,106,3,0,124,3,0,116,2,0, - 106,4,0,116,2,0,106,5,0,66,116,2,0,106,6,0, - 66,124,2,0,100,2,0,64,131,3,0,125,4,0,121,61, - 0,116,7,0,106,8,0,124,4,0,100,3,0,131,2,0, - 143,20,0,125,5,0,124,5,0,106,9,0,124,1,0,131, - 1,0,1,87,100,4,0,81,82,88,116,2,0,106,10,0, - 124,3,0,124,0,0,131,2,0,1,87,110,59,0,4,116, - 11,0,107,10,0,114,188,0,1,1,1,121,17,0,116,2, - 0,106,12,0,124,3,0,131,1,0,1,87,110,18,0,4, - 116,11,0,107,10,0,114,180,0,1,1,1,89,110,1,0, - 88,130,0,0,89,110,1,0,88,100,4,0,83,41,5,122, - 162,66,101,115,116,45,101,102,102,111,114,116,32,102,117,110, - 99,116,105,111,110,32,116,111,32,119,114,105,116,101,32,100, - 97,116,97,32,116,111,32,97,32,112,97,116,104,32,97,116, - 111,109,105,99,97,108,108,121,46,10,32,32,32,32,66,101, - 32,112,114,101,112,97,114,101,100,32,116,111,32,104,97,110, - 100,108,101,32,97,32,70,105,108,101,69,120,105,115,116,115, - 69,114,114,111,114,32,105,102,32,99,111,110,99,117,114,114, - 101,110,116,32,119,114,105,116,105,110,103,32,111,102,32,116, - 104,101,10,32,32,32,32,116,101,109,112,111,114,97,114,121, - 32,102,105,108,101,32,105,115,32,97,116,116,101,109,112,116, - 101,100,46,122,5,123,125,46,123,125,105,182,1,0,0,90, - 2,119,98,78,41,13,218,6,102,111,114,109,97,116,218,2, - 105,100,114,3,0,0,0,90,4,111,112,101,110,90,6,79, - 95,69,88,67,76,90,7,79,95,67,82,69,65,84,90,8, - 79,95,87,82,79,78,76,89,218,3,95,105,111,218,6,70, - 105,108,101,73,79,218,5,119,114,105,116,101,218,7,114,101, - 112,108,97,99,101,114,40,0,0,0,90,6,117,110,108,105, - 110,107,41,6,114,35,0,0,0,218,4,100,97,116,97,114, - 42,0,0,0,90,8,112,97,116,104,95,116,109,112,90,2, - 102,100,218,4,102,105,108,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,13,95,119,114,105,116,101,95, - 97,116,111,109,105,99,99,0,0,0,115,26,0,0,0,0, - 5,24,1,9,1,33,1,3,3,21,1,20,1,20,1,13, - 1,3,1,17,1,13,1,5,1,114,55,0,0,0,105,22, - 13,0,0,233,2,0,0,0,114,13,0,0,0,115,2,0, - 0,0,13,10,90,11,95,95,112,121,99,97,99,104,101,95, - 95,122,4,111,112,116,45,122,3,46,112,121,122,4,46,112, - 121,99,78,218,12,111,112,116,105,109,105,122,97,116,105,111, - 110,99,2,0,0,0,1,0,0,0,11,0,0,0,6,0, - 0,0,67,0,0,0,115,87,1,0,0,124,1,0,100,1, - 0,107,9,0,114,76,0,116,0,0,106,1,0,100,2,0, - 116,2,0,131,2,0,1,124,2,0,100,1,0,107,9,0, - 114,58,0,100,3,0,125,3,0,116,3,0,124,3,0,131, - 1,0,130,1,0,124,1,0,114,70,0,100,4,0,110,3, - 0,100,5,0,125,2,0,116,4,0,124,0,0,131,1,0, - 92,2,0,125,4,0,125,5,0,124,5,0,106,5,0,100, - 6,0,131,1,0,92,3,0,125,6,0,125,7,0,125,8, - 0,116,6,0,106,7,0,106,8,0,125,9,0,124,9,0, - 100,1,0,107,8,0,114,154,0,116,9,0,100,7,0,131, - 1,0,130,1,0,100,4,0,106,10,0,124,6,0,114,172, - 0,124,6,0,110,3,0,124,8,0,124,7,0,124,9,0, - 103,3,0,131,1,0,125,10,0,124,2,0,100,1,0,107, - 8,0,114,241,0,116,6,0,106,11,0,106,12,0,100,8, - 0,107,2,0,114,229,0,100,4,0,125,2,0,110,12,0, - 116,6,0,106,11,0,106,12,0,125,2,0,116,13,0,124, - 2,0,131,1,0,125,2,0,124,2,0,100,4,0,107,3, - 0,114,63,1,124,2,0,106,14,0,131,0,0,115,42,1, - 116,15,0,100,9,0,106,16,0,124,2,0,131,1,0,131, - 1,0,130,1,0,100,10,0,106,16,0,124,10,0,116,17, - 0,124,2,0,131,3,0,125,10,0,116,18,0,124,4,0, - 116,19,0,124,10,0,116,20,0,100,8,0,25,23,131,3, - 0,83,41,11,97,254,2,0,0,71,105,118,101,110,32,116, - 104,101,32,112,97,116,104,32,116,111,32,97,32,46,112,121, - 32,102,105,108,101,44,32,114,101,116,117,114,110,32,116,104, - 101,32,112,97,116,104,32,116,111,32,105,116,115,32,46,112, - 121,99,32,102,105,108,101,46,10,10,32,32,32,32,84,104, - 101,32,46,112,121,32,102,105,108,101,32,100,111,101,115,32, - 110,111,116,32,110,101,101,100,32,116,111,32,101,120,105,115, - 116,59,32,116,104,105,115,32,115,105,109,112,108,121,32,114, - 101,116,117,114,110,115,32,116,104,101,32,112,97,116,104,32, - 116,111,32,116,104,101,10,32,32,32,32,46,112,121,99,32, - 102,105,108,101,32,99,97,108,99,117,108,97,116,101,100,32, - 97,115,32,105,102,32,116,104,101,32,46,112,121,32,102,105, - 108,101,32,119,101,114,101,32,105,109,112,111,114,116,101,100, - 46,10,10,32,32,32,32,84,104,101,32,39,111,112,116,105, - 109,105,122,97,116,105,111,110,39,32,112,97,114,97,109,101, - 116,101,114,32,99,111,110,116,114,111,108,115,32,116,104,101, - 32,112,114,101,115,117,109,101,100,32,111,112,116,105,109,105, - 122,97,116,105,111,110,32,108,101,118,101,108,32,111,102,10, - 32,32,32,32,116,104,101,32,98,121,116,101,99,111,100,101, - 32,102,105,108,101,46,32,73,102,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,105,115,32,110,111,116,32, - 78,111,110,101,44,32,116,104,101,32,115,116,114,105,110,103, - 32,114,101,112,114,101,115,101,110,116,97,116,105,111,110,10, - 32,32,32,32,111,102,32,116,104,101,32,97,114,103,117,109, - 101,110,116,32,105,115,32,116,97,107,101,110,32,97,110,100, - 32,118,101,114,105,102,105,101,100,32,116,111,32,98,101,32, - 97,108,112,104,97,110,117,109,101,114,105,99,32,40,101,108, - 115,101,32,86,97,108,117,101,69,114,114,111,114,10,32,32, - 32,32,105,115,32,114,97,105,115,101,100,41,46,10,10,32, - 32,32,32,84,104,101,32,100,101,98,117,103,95,111,118,101, - 114,114,105,100,101,32,112,97,114,97,109,101,116,101,114,32, - 105,115,32,100,101,112,114,101,99,97,116,101,100,46,32,73, - 102,32,100,101,98,117,103,95,111,118,101,114,114,105,100,101, - 32,105,115,32,110,111,116,32,78,111,110,101,44,10,32,32, - 32,32,97,32,84,114,117,101,32,118,97,108,117,101,32,105, - 115,32,116,104,101,32,115,97,109,101,32,97,115,32,115,101, - 116,116,105,110,103,32,39,111,112,116,105,109,105,122,97,116, - 105,111,110,39,32,116,111,32,116,104,101,32,101,109,112,116, - 121,32,115,116,114,105,110,103,10,32,32,32,32,119,104,105, - 108,101,32,97,32,70,97,108,115,101,32,118,97,108,117,101, - 32,105,115,32,101,113,117,105,118,97,108,101,110,116,32,116, - 111,32,115,101,116,116,105,110,103,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,116,111,32,39,49,39,46, - 10,10,32,32,32,32,73,102,32,115,121,115,46,105,109,112, - 108,101,109,101,110,116,97,116,105,111,110,46,99,97,99,104, - 101,95,116,97,103,32,105,115,32,78,111,110,101,32,116,104, - 101,110,32,78,111,116,73,109,112,108,101,109,101,110,116,101, - 100,69,114,114,111,114,32,105,115,32,114,97,105,115,101,100, - 46,10,10,32,32,32,32,78,122,70,116,104,101,32,100,101, - 98,117,103,95,111,118,101,114,114,105,100,101,32,112,97,114, - 97,109,101,116,101,114,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,59,32,117,115,101,32,39,111,112,116,105,109, - 105,122,97,116,105,111,110,39,32,105,110,115,116,101,97,100, - 122,50,100,101,98,117,103,95,111,118,101,114,114,105,100,101, - 32,111,114,32,111,112,116,105,109,105,122,97,116,105,111,110, - 32,109,117,115,116,32,98,101,32,115,101,116,32,116,111,32, - 78,111,110,101,114,30,0,0,0,114,29,0,0,0,218,1, - 46,122,36,115,121,115,46,105,109,112,108,101,109,101,110,116, - 97,116,105,111,110,46,99,97,99,104,101,95,116,97,103,32, - 105,115,32,78,111,110,101,233,0,0,0,0,122,24,123,33, - 114,125,32,105,115,32,110,111,116,32,97,108,112,104,97,110, - 117,109,101,114,105,99,122,7,123,125,46,123,125,123,125,41, - 21,218,9,95,119,97,114,110,105,110,103,115,218,4,119,97, - 114,110,218,18,68,101,112,114,101,99,97,116,105,111,110,87, - 97,114,110,105,110,103,218,9,84,121,112,101,69,114,114,111, - 114,114,38,0,0,0,114,32,0,0,0,114,7,0,0,0, - 218,14,105,109,112,108,101,109,101,110,116,97,116,105,111,110, - 218,9,99,97,99,104,101,95,116,97,103,218,19,78,111,116, - 73,109,112,108,101,109,101,110,116,101,100,69,114,114,111,114, - 114,26,0,0,0,218,5,102,108,97,103,115,218,8,111,112, - 116,105,109,105,122,101,218,3,115,116,114,218,7,105,115,97, - 108,110,117,109,218,10,86,97,108,117,101,69,114,114,111,114, - 114,47,0,0,0,218,4,95,79,80,84,114,28,0,0,0, - 218,8,95,80,89,67,65,67,72,69,218,17,66,89,84,69, - 67,79,68,69,95,83,85,70,70,73,88,69,83,41,11,114, - 35,0,0,0,90,14,100,101,98,117,103,95,111,118,101,114, - 114,105,100,101,114,57,0,0,0,218,7,109,101,115,115,97, - 103,101,218,4,104,101,97,100,114,37,0,0,0,90,4,98, - 97,115,101,218,3,115,101,112,218,4,114,101,115,116,90,3, - 116,97,103,90,15,97,108,109,111,115,116,95,102,105,108,101, - 110,97,109,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,17,99,97,99,104,101,95,102,114,111,109,95, - 115,111,117,114,99,101,243,0,0,0,115,46,0,0,0,0, - 18,12,1,9,1,7,1,12,1,6,1,12,1,18,1,18, - 1,24,1,12,1,12,1,12,1,36,1,12,1,18,1,9, - 2,12,1,12,1,12,1,12,1,21,1,21,1,114,79,0, - 0,0,99,1,0,0,0,0,0,0,0,8,0,0,0,5, - 0,0,0,67,0,0,0,115,62,1,0,0,116,0,0,106, - 1,0,106,2,0,100,1,0,107,8,0,114,30,0,116,3, - 0,100,2,0,131,1,0,130,1,0,116,4,0,124,0,0, - 131,1,0,92,2,0,125,1,0,125,2,0,116,4,0,124, - 1,0,131,1,0,92,2,0,125,1,0,125,3,0,124,3, - 0,116,5,0,107,3,0,114,102,0,116,6,0,100,3,0, - 106,7,0,116,5,0,124,0,0,131,2,0,131,1,0,130, - 1,0,124,2,0,106,8,0,100,4,0,131,1,0,125,4, - 0,124,4,0,100,11,0,107,7,0,114,153,0,116,6,0, - 100,7,0,106,7,0,124,2,0,131,1,0,131,1,0,130, - 1,0,110,125,0,124,4,0,100,6,0,107,2,0,114,22, - 1,124,2,0,106,9,0,100,4,0,100,5,0,131,2,0, - 100,12,0,25,125,5,0,124,5,0,106,10,0,116,11,0, - 131,1,0,115,223,0,116,6,0,100,8,0,106,7,0,116, - 11,0,131,1,0,131,1,0,130,1,0,124,5,0,116,12, - 0,116,11,0,131,1,0,100,1,0,133,2,0,25,125,6, - 0,124,6,0,106,13,0,131,0,0,115,22,1,116,6,0, - 100,9,0,106,7,0,124,5,0,131,1,0,131,1,0,130, - 1,0,124,2,0,106,14,0,100,4,0,131,1,0,100,10, - 0,25,125,7,0,116,15,0,124,1,0,124,7,0,116,16, - 0,100,10,0,25,23,131,2,0,83,41,13,97,110,1,0, - 0,71,105,118,101,110,32,116,104,101,32,112,97,116,104,32, - 116,111,32,97,32,46,112,121,99,46,32,102,105,108,101,44, - 32,114,101,116,117,114,110,32,116,104,101,32,112,97,116,104, - 32,116,111,32,105,116,115,32,46,112,121,32,102,105,108,101, - 46,10,10,32,32,32,32,84,104,101,32,46,112,121,99,32, - 102,105,108,101,32,100,111,101,115,32,110,111,116,32,110,101, - 101,100,32,116,111,32,101,120,105,115,116,59,32,116,104,105, - 115,32,115,105,109,112,108,121,32,114,101,116,117,114,110,115, - 32,116,104,101,32,112,97,116,104,32,116,111,10,32,32,32, - 32,116,104,101,32,46,112,121,32,102,105,108,101,32,99,97, - 108,99,117,108,97,116,101,100,32,116,111,32,99,111,114,114, - 101,115,112,111,110,100,32,116,111,32,116,104,101,32,46,112, - 121,99,32,102,105,108,101,46,32,32,73,102,32,112,97,116, - 104,32,100,111,101,115,10,32,32,32,32,110,111,116,32,99, - 111,110,102,111,114,109,32,116,111,32,80,69,80,32,51,49, - 52,55,47,52,56,56,32,102,111,114,109,97,116,44,32,86, - 97,108,117,101,69,114,114,111,114,32,119,105,108,108,32,98, - 101,32,114,97,105,115,101,100,46,32,73,102,10,32,32,32, - 32,115,121,115,46,105,109,112,108,101,109,101,110,116,97,116, + 104,95,105,115,100,105,114,92,0,0,0,115,6,0,0,0, + 0,2,6,1,12,1,114,46,0,0,0,105,182,1,0,0, + 99,3,0,0,0,0,0,0,0,6,0,0,0,17,0,0, + 0,67,0,0,0,115,193,0,0,0,100,1,0,106,0,0, + 124,0,0,116,1,0,124,0,0,131,1,0,131,2,0,125, + 3,0,116,2,0,106,3,0,124,3,0,116,2,0,106,4, + 0,116,2,0,106,5,0,66,116,2,0,106,6,0,66,124, + 2,0,100,2,0,64,131,3,0,125,4,0,121,61,0,116, + 7,0,106,8,0,124,4,0,100,3,0,131,2,0,143,20, + 0,125,5,0,124,5,0,106,9,0,124,1,0,131,1,0, + 1,87,100,4,0,81,82,88,116,2,0,106,10,0,124,3, + 0,124,0,0,131,2,0,1,87,110,59,0,4,116,11,0, + 107,10,0,114,188,0,1,1,1,121,17,0,116,2,0,106, + 12,0,124,3,0,131,1,0,1,87,110,18,0,4,116,11, + 0,107,10,0,114,180,0,1,1,1,89,110,1,0,88,130, + 0,0,89,110,1,0,88,100,4,0,83,41,5,122,162,66, + 101,115,116,45,101,102,102,111,114,116,32,102,117,110,99,116, + 105,111,110,32,116,111,32,119,114,105,116,101,32,100,97,116, + 97,32,116,111,32,97,32,112,97,116,104,32,97,116,111,109, + 105,99,97,108,108,121,46,10,32,32,32,32,66,101,32,112, + 114,101,112,97,114,101,100,32,116,111,32,104,97,110,100,108, + 101,32,97,32,70,105,108,101,69,120,105,115,116,115,69,114, + 114,111,114,32,105,102,32,99,111,110,99,117,114,114,101,110, + 116,32,119,114,105,116,105,110,103,32,111,102,32,116,104,101, + 10,32,32,32,32,116,101,109,112,111,114,97,114,121,32,102, + 105,108,101,32,105,115,32,97,116,116,101,109,112,116,101,100, + 46,122,5,123,125,46,123,125,105,182,1,0,0,90,2,119, + 98,78,41,13,218,6,102,111,114,109,97,116,218,2,105,100, + 114,3,0,0,0,90,4,111,112,101,110,90,6,79,95,69, + 88,67,76,90,7,79,95,67,82,69,65,84,90,8,79,95, + 87,82,79,78,76,89,218,3,95,105,111,218,6,70,105,108, + 101,73,79,218,5,119,114,105,116,101,218,7,114,101,112,108, + 97,99,101,114,40,0,0,0,90,6,117,110,108,105,110,107, + 41,6,114,35,0,0,0,218,4,100,97,116,97,114,42,0, + 0,0,90,8,112,97,116,104,95,116,109,112,90,2,102,100, + 218,4,102,105,108,101,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, + 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, + 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, + 1,17,1,13,1,5,1,114,55,0,0,0,105,22,13,0, + 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, + 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, + 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, + 78,218,12,111,112,116,105,109,105,122,97,116,105,111,110,99, + 2,0,0,0,1,0,0,0,11,0,0,0,6,0,0,0, + 67,0,0,0,115,87,1,0,0,124,1,0,100,1,0,107, + 9,0,114,76,0,116,0,0,106,1,0,100,2,0,116,2, + 0,131,2,0,1,124,2,0,100,1,0,107,9,0,114,58, + 0,100,3,0,125,3,0,116,3,0,124,3,0,131,1,0, + 130,1,0,124,1,0,114,70,0,100,4,0,110,3,0,100, + 5,0,125,2,0,116,4,0,124,0,0,131,1,0,92,2, + 0,125,4,0,125,5,0,124,5,0,106,5,0,100,6,0, + 131,1,0,92,3,0,125,6,0,125,7,0,125,8,0,116, + 6,0,106,7,0,106,8,0,125,9,0,124,9,0,100,1, + 0,107,8,0,114,154,0,116,9,0,100,7,0,131,1,0, + 130,1,0,100,4,0,106,10,0,124,6,0,114,172,0,124, + 6,0,110,3,0,124,8,0,124,7,0,124,9,0,103,3, + 0,131,1,0,125,10,0,124,2,0,100,1,0,107,8,0, + 114,241,0,116,6,0,106,11,0,106,12,0,100,8,0,107, + 2,0,114,229,0,100,4,0,125,2,0,110,12,0,116,6, + 0,106,11,0,106,12,0,125,2,0,116,13,0,124,2,0, + 131,1,0,125,2,0,124,2,0,100,4,0,107,3,0,114, + 63,1,124,2,0,106,14,0,131,0,0,115,42,1,116,15, + 0,100,9,0,106,16,0,124,2,0,131,1,0,131,1,0, + 130,1,0,100,10,0,106,16,0,124,10,0,116,17,0,124, + 2,0,131,3,0,125,10,0,116,18,0,124,4,0,116,19, + 0,124,10,0,116,20,0,100,8,0,25,23,131,3,0,83, + 41,11,97,254,2,0,0,71,105,118,101,110,32,116,104,101, + 32,112,97,116,104,32,116,111,32,97,32,46,112,121,32,102, + 105,108,101,44,32,114,101,116,117,114,110,32,116,104,101,32, + 112,97,116,104,32,116,111,32,105,116,115,32,46,112,121,99, + 32,102,105,108,101,46,10,10,32,32,32,32,84,104,101,32, + 46,112,121,32,102,105,108,101,32,100,111,101,115,32,110,111, + 116,32,110,101,101,100,32,116,111,32,101,120,105,115,116,59, + 32,116,104,105,115,32,115,105,109,112,108,121,32,114,101,116, + 117,114,110,115,32,116,104,101,32,112,97,116,104,32,116,111, + 32,116,104,101,10,32,32,32,32,46,112,121,99,32,102,105, + 108,101,32,99,97,108,99,117,108,97,116,101,100,32,97,115, + 32,105,102,32,116,104,101,32,46,112,121,32,102,105,108,101, + 32,119,101,114,101,32,105,109,112,111,114,116,101,100,46,10, + 10,32,32,32,32,84,104,101,32,39,111,112,116,105,109,105, + 122,97,116,105,111,110,39,32,112,97,114,97,109,101,116,101, + 114,32,99,111,110,116,114,111,108,115,32,116,104,101,32,112, + 114,101,115,117,109,101,100,32,111,112,116,105,109,105,122,97, + 116,105,111,110,32,108,101,118,101,108,32,111,102,10,32,32, + 32,32,116,104,101,32,98,121,116,101,99,111,100,101,32,102, + 105,108,101,46,32,73,102,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,105,115,32,110,111,116,32,78,111, + 110,101,44,32,116,104,101,32,115,116,114,105,110,103,32,114, + 101,112,114,101,115,101,110,116,97,116,105,111,110,10,32,32, + 32,32,111,102,32,116,104,101,32,97,114,103,117,109,101,110, + 116,32,105,115,32,116,97,107,101,110,32,97,110,100,32,118, + 101,114,105,102,105,101,100,32,116,111,32,98,101,32,97,108, + 112,104,97,110,117,109,101,114,105,99,32,40,101,108,115,101, + 32,86,97,108,117,101,69,114,114,111,114,10,32,32,32,32, + 105,115,32,114,97,105,115,101,100,41,46,10,10,32,32,32, + 32,84,104,101,32,100,101,98,117,103,95,111,118,101,114,114, + 105,100,101,32,112,97,114,97,109,101,116,101,114,32,105,115, + 32,100,101,112,114,101,99,97,116,101,100,46,32,73,102,32, + 100,101,98,117,103,95,111,118,101,114,114,105,100,101,32,105, + 115,32,110,111,116,32,78,111,110,101,44,10,32,32,32,32, + 97,32,84,114,117,101,32,118,97,108,117,101,32,105,115,32, + 116,104,101,32,115,97,109,101,32,97,115,32,115,101,116,116, + 105,110,103,32,39,111,112,116,105,109,105,122,97,116,105,111, + 110,39,32,116,111,32,116,104,101,32,101,109,112,116,121,32, + 115,116,114,105,110,103,10,32,32,32,32,119,104,105,108,101, + 32,97,32,70,97,108,115,101,32,118,97,108,117,101,32,105, + 115,32,101,113,117,105,118,97,108,101,110,116,32,116,111,32, + 115,101,116,116,105,110,103,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,116,111,32,39,49,39,46,10,10, + 32,32,32,32,73,102,32,115,121,115,46,105,109,112,108,101, + 109,101,110,116,97,116,105,111,110,46,99,97,99,104,101,95, + 116,97,103,32,105,115,32,78,111,110,101,32,116,104,101,110, + 32,78,111,116,73,109,112,108,101,109,101,110,116,101,100,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,46,10, + 10,32,32,32,32,78,122,70,116,104,101,32,100,101,98,117, + 103,95,111,118,101,114,114,105,100,101,32,112,97,114,97,109, + 101,116,101,114,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,59,32,117,115,101,32,39,111,112,116,105,109,105,122, + 97,116,105,111,110,39,32,105,110,115,116,101,97,100,122,50, + 100,101,98,117,103,95,111,118,101,114,114,105,100,101,32,111, + 114,32,111,112,116,105,109,105,122,97,116,105,111,110,32,109, + 117,115,116,32,98,101,32,115,101,116,32,116,111,32,78,111, + 110,101,114,30,0,0,0,114,29,0,0,0,218,1,46,122, + 36,115,121,115,46,105,109,112,108,101,109,101,110,116,97,116, 105,111,110,46,99,97,99,104,101,95,116,97,103,32,105,115, - 32,78,111,110,101,32,116,104,101,110,32,78,111,116,73,109, - 112,108,101,109,101,110,116,101,100,69,114,114,111,114,32,105, - 115,32,114,97,105,115,101,100,46,10,10,32,32,32,32,78, - 122,36,115,121,115,46,105,109,112,108,101,109,101,110,116,97, - 116,105,111,110,46,99,97,99,104,101,95,116,97,103,32,105, - 115,32,78,111,110,101,122,37,123,125,32,110,111,116,32,98, - 111,116,116,111,109,45,108,101,118,101,108,32,100,105,114,101, - 99,116,111,114,121,32,105,110,32,123,33,114,125,114,58,0, - 0,0,114,56,0,0,0,233,3,0,0,0,122,33,101,120, - 112,101,99,116,101,100,32,111,110,108,121,32,50,32,111,114, - 32,51,32,100,111,116,115,32,105,110,32,123,33,114,125,122, - 57,111,112,116,105,109,105,122,97,116,105,111,110,32,112,111, - 114,116,105,111,110,32,111,102,32,102,105,108,101,110,97,109, - 101,32,100,111,101,115,32,110,111,116,32,115,116,97,114,116, - 32,119,105,116,104,32,123,33,114,125,122,52,111,112,116,105, - 109,105,122,97,116,105,111,110,32,108,101,118,101,108,32,123, - 33,114,125,32,105,115,32,110,111,116,32,97,110,32,97,108, - 112,104,97,110,117,109,101,114,105,99,32,118,97,108,117,101, - 114,59,0,0,0,62,2,0,0,0,114,56,0,0,0,114, - 80,0,0,0,233,254,255,255,255,41,17,114,7,0,0,0, - 114,64,0,0,0,114,65,0,0,0,114,66,0,0,0,114, - 38,0,0,0,114,73,0,0,0,114,71,0,0,0,114,47, - 0,0,0,218,5,99,111,117,110,116,114,34,0,0,0,114, - 9,0,0,0,114,72,0,0,0,114,31,0,0,0,114,70, - 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,28, - 0,0,0,218,15,83,79,85,82,67,69,95,83,85,70,70, - 73,88,69,83,41,8,114,35,0,0,0,114,76,0,0,0, - 90,16,112,121,99,97,99,104,101,95,102,105,108,101,110,97, - 109,101,90,7,112,121,99,97,99,104,101,90,9,100,111,116, - 95,99,111,117,110,116,114,57,0,0,0,90,9,111,112,116, - 95,108,101,118,101,108,90,13,98,97,115,101,95,102,105,108, - 101,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,115,111,117,114,99,101,95,102,114,111, - 109,95,99,97,99,104,101,31,1,0,0,115,44,0,0,0, - 0,9,18,1,12,1,18,1,18,1,12,1,9,1,15,1, - 15,1,12,1,9,1,15,1,12,1,22,1,15,1,9,1, - 12,1,22,1,12,1,9,1,12,1,19,1,114,85,0,0, - 0,99,1,0,0,0,0,0,0,0,5,0,0,0,12,0, - 0,0,67,0,0,0,115,164,0,0,0,116,0,0,124,0, - 0,131,1,0,100,1,0,107,2,0,114,22,0,100,2,0, - 83,124,0,0,106,1,0,100,3,0,131,1,0,92,3,0, - 125,1,0,125,2,0,125,3,0,124,1,0,12,115,81,0, - 124,3,0,106,2,0,131,0,0,100,7,0,100,8,0,133, - 2,0,25,100,6,0,107,3,0,114,85,0,124,0,0,83, - 121,16,0,116,3,0,124,0,0,131,1,0,125,4,0,87, - 110,40,0,4,116,4,0,116,5,0,102,2,0,107,10,0, - 114,143,0,1,1,1,124,0,0,100,2,0,100,9,0,133, - 2,0,25,125,4,0,89,110,1,0,88,116,6,0,124,4, - 0,131,1,0,114,160,0,124,4,0,83,124,0,0,83,41, - 10,122,188,67,111,110,118,101,114,116,32,97,32,98,121,116, - 101,99,111,100,101,32,102,105,108,101,32,112,97,116,104,32, - 116,111,32,97,32,115,111,117,114,99,101,32,112,97,116,104, - 32,40,105,102,32,112,111,115,115,105,98,108,101,41,46,10, - 10,32,32,32,32,84,104,105,115,32,102,117,110,99,116,105, - 111,110,32,101,120,105,115,116,115,32,112,117,114,101,108,121, - 32,102,111,114,32,98,97,99,107,119,97,114,100,115,45,99, - 111,109,112,97,116,105,98,105,108,105,116,121,32,102,111,114, - 10,32,32,32,32,80,121,73,109,112,111,114,116,95,69,120, - 101,99,67,111,100,101,77,111,100,117,108,101,87,105,116,104, - 70,105,108,101,110,97,109,101,115,40,41,32,105,110,32,116, - 104,101,32,67,32,65,80,73,46,10,10,32,32,32,32,114, - 59,0,0,0,78,114,58,0,0,0,114,80,0,0,0,114, - 29,0,0,0,90,2,112,121,233,253,255,255,255,233,255,255, - 255,255,114,87,0,0,0,41,7,114,31,0,0,0,114,32, - 0,0,0,218,5,108,111,119,101,114,114,85,0,0,0,114, - 66,0,0,0,114,71,0,0,0,114,44,0,0,0,41,5, - 218,13,98,121,116,101,99,111,100,101,95,112,97,116,104,114, - 78,0,0,0,114,36,0,0,0,90,9,101,120,116,101,110, - 115,105,111,110,218,11,115,111,117,114,99,101,95,112,97,116, - 104,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,15,95,103,101,116,95,115,111,117,114,99,101,102,105,108, - 101,64,1,0,0,115,20,0,0,0,0,7,18,1,4,1, - 24,1,35,1,4,1,3,1,16,1,19,1,21,1,114,91, - 0,0,0,99,1,0,0,0,0,0,0,0,1,0,0,0, - 11,0,0,0,67,0,0,0,115,92,0,0,0,124,0,0, - 106,0,0,116,1,0,116,2,0,131,1,0,131,1,0,114, - 59,0,121,14,0,116,3,0,124,0,0,131,1,0,83,87, - 113,88,0,4,116,4,0,107,10,0,114,55,0,1,1,1, - 89,113,88,0,88,110,29,0,124,0,0,106,0,0,116,1, - 0,116,5,0,131,1,0,131,1,0,114,84,0,124,0,0, - 83,100,0,0,83,100,0,0,83,41,1,78,41,6,218,8, - 101,110,100,115,119,105,116,104,218,5,116,117,112,108,101,114, - 84,0,0,0,114,79,0,0,0,114,66,0,0,0,114,74, - 0,0,0,41,1,218,8,102,105,108,101,110,97,109,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 95,103,101,116,95,99,97,99,104,101,100,83,1,0,0,115, - 16,0,0,0,0,1,21,1,3,1,14,1,13,1,8,1, - 21,1,4,2,114,95,0,0,0,99,1,0,0,0,0,0, - 0,0,2,0,0,0,11,0,0,0,67,0,0,0,115,60, - 0,0,0,121,19,0,116,0,0,124,0,0,131,1,0,106, - 1,0,125,1,0,87,110,24,0,4,116,2,0,107,10,0, - 114,45,0,1,1,1,100,1,0,125,1,0,89,110,1,0, - 88,124,1,0,100,2,0,79,125,1,0,124,1,0,83,41, - 3,122,51,67,97,108,99,117,108,97,116,101,32,116,104,101, - 32,109,111,100,101,32,112,101,114,109,105,115,115,105,111,110, - 115,32,102,111,114,32,97,32,98,121,116,101,99,111,100,101, - 32,102,105,108,101,46,105,182,1,0,0,233,128,0,0,0, - 41,3,114,39,0,0,0,114,41,0,0,0,114,40,0,0, - 0,41,2,114,35,0,0,0,114,42,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,10,95,99, - 97,108,99,95,109,111,100,101,95,1,0,0,115,12,0,0, - 0,0,2,3,1,19,1,13,1,11,3,10,1,114,97,0, - 0,0,218,9,118,101,114,98,111,115,105,116,121,114,29,0, - 0,0,99,1,0,0,0,1,0,0,0,3,0,0,0,4, - 0,0,0,71,0,0,0,115,75,0,0,0,116,0,0,106, - 1,0,106,2,0,124,1,0,107,5,0,114,71,0,124,0, - 0,106,3,0,100,6,0,131,1,0,115,43,0,100,3,0, - 124,0,0,23,125,0,0,116,4,0,124,0,0,106,5,0, - 124,2,0,140,0,0,100,4,0,116,0,0,106,6,0,131, - 1,1,1,100,5,0,83,41,7,122,61,80,114,105,110,116, - 32,116,104,101,32,109,101,115,115,97,103,101,32,116,111,32, - 115,116,100,101,114,114,32,105,102,32,45,118,47,80,89,84, - 72,79,78,86,69,82,66,79,83,69,32,105,115,32,116,117, - 114,110,101,100,32,111,110,46,250,1,35,250,7,105,109,112, - 111,114,116,32,122,2,35,32,114,54,0,0,0,78,41,2, - 114,99,0,0,0,114,100,0,0,0,41,7,114,7,0,0, - 0,114,67,0,0,0,218,7,118,101,114,98,111,115,101,114, - 9,0,0,0,218,5,112,114,105,110,116,114,47,0,0,0, - 218,6,115,116,100,101,114,114,41,3,114,75,0,0,0,114, - 98,0,0,0,218,4,97,114,103,115,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,16,95,118,101,114,98, - 111,115,101,95,109,101,115,115,97,103,101,107,1,0,0,115, - 8,0,0,0,0,2,18,1,15,1,10,1,114,105,0,0, - 0,99,1,0,0,0,0,0,0,0,3,0,0,0,11,0, - 0,0,3,0,0,0,115,84,0,0,0,100,1,0,135,0, - 0,102,1,0,100,2,0,100,3,0,134,1,0,125,1,0, - 121,13,0,116,0,0,106,1,0,125,2,0,87,110,30,0, - 4,116,2,0,107,10,0,114,66,0,1,1,1,100,4,0, - 100,5,0,132,0,0,125,2,0,89,110,1,0,88,124,2, - 0,124,1,0,136,0,0,131,2,0,1,124,1,0,83,41, - 6,122,252,68,101,99,111,114,97,116,111,114,32,116,111,32, - 118,101,114,105,102,121,32,116,104,97,116,32,116,104,101,32, - 109,111,100,117,108,101,32,98,101,105,110,103,32,114,101,113, - 117,101,115,116,101,100,32,109,97,116,99,104,101,115,32,116, - 104,101,32,111,110,101,32,116,104,101,10,32,32,32,32,108, - 111,97,100,101,114,32,99,97,110,32,104,97,110,100,108,101, - 46,10,10,32,32,32,32,84,104,101,32,102,105,114,115,116, - 32,97,114,103,117,109,101,110,116,32,40,115,101,108,102,41, - 32,109,117,115,116,32,100,101,102,105,110,101,32,95,110,97, - 109,101,32,119,104,105,99,104,32,116,104,101,32,115,101,99, - 111,110,100,32,97,114,103,117,109,101,110,116,32,105,115,10, - 32,32,32,32,99,111,109,112,97,114,101,100,32,97,103,97, - 105,110,115,116,46,32,73,102,32,116,104,101,32,99,111,109, - 112,97,114,105,115,111,110,32,102,97,105,108,115,32,116,104, - 101,110,32,73,109,112,111,114,116,69,114,114,111,114,32,105, - 115,32,114,97,105,115,101,100,46,10,10,32,32,32,32,78, - 99,2,0,0,0,0,0,0,0,4,0,0,0,5,0,0, - 0,31,0,0,0,115,89,0,0,0,124,1,0,100,0,0, - 107,8,0,114,24,0,124,0,0,106,0,0,125,1,0,110, - 46,0,124,0,0,106,0,0,124,1,0,107,3,0,114,70, - 0,116,1,0,100,1,0,124,0,0,106,0,0,124,1,0, - 102,2,0,22,100,2,0,124,1,0,131,1,1,130,1,0, - 136,0,0,124,0,0,124,1,0,124,2,0,124,3,0,142, - 2,0,83,41,3,78,122,30,108,111,97,100,101,114,32,102, - 111,114,32,37,115,32,99,97,110,110,111,116,32,104,97,110, - 100,108,101,32,37,115,218,4,110,97,109,101,41,2,114,106, - 0,0,0,218,11,73,109,112,111,114,116,69,114,114,111,114, - 41,4,218,4,115,101,108,102,114,106,0,0,0,114,104,0, - 0,0,90,6,107,119,97,114,103,115,41,1,218,6,109,101, + 32,78,111,110,101,233,0,0,0,0,122,24,123,33,114,125, + 32,105,115,32,110,111,116,32,97,108,112,104,97,110,117,109, + 101,114,105,99,122,7,123,125,46,123,125,123,125,41,21,218, + 9,95,119,97,114,110,105,110,103,115,218,4,119,97,114,110, + 218,18,68,101,112,114,101,99,97,116,105,111,110,87,97,114, + 110,105,110,103,218,9,84,121,112,101,69,114,114,111,114,114, + 38,0,0,0,114,32,0,0,0,114,7,0,0,0,218,14, + 105,109,112,108,101,109,101,110,116,97,116,105,111,110,218,9, + 99,97,99,104,101,95,116,97,103,218,19,78,111,116,73,109, + 112,108,101,109,101,110,116,101,100,69,114,114,111,114,114,26, + 0,0,0,218,5,102,108,97,103,115,218,8,111,112,116,105, + 109,105,122,101,218,3,115,116,114,218,7,105,115,97,108,110, + 117,109,218,10,86,97,108,117,101,69,114,114,111,114,114,47, + 0,0,0,218,4,95,79,80,84,114,28,0,0,0,218,8, + 95,80,89,67,65,67,72,69,218,17,66,89,84,69,67,79, + 68,69,95,83,85,70,70,73,88,69,83,41,11,114,35,0, + 0,0,90,14,100,101,98,117,103,95,111,118,101,114,114,105, + 100,101,114,57,0,0,0,218,7,109,101,115,115,97,103,101, + 218,4,104,101,97,100,114,37,0,0,0,90,4,98,97,115, + 101,218,3,115,101,112,218,4,114,101,115,116,90,3,116,97, + 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, + 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, + 117,114,99,101,243,0,0,0,115,46,0,0,0,0,18,12, + 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, + 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, + 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, + 99,1,0,0,0,0,0,0,0,8,0,0,0,5,0,0, + 0,67,0,0,0,115,62,1,0,0,116,0,0,106,1,0, + 106,2,0,100,1,0,107,8,0,114,30,0,116,3,0,100, + 2,0,131,1,0,130,1,0,116,4,0,124,0,0,131,1, + 0,92,2,0,125,1,0,125,2,0,116,4,0,124,1,0, + 131,1,0,92,2,0,125,1,0,125,3,0,124,3,0,116, + 5,0,107,3,0,114,102,0,116,6,0,100,3,0,106,7, + 0,116,5,0,124,0,0,131,2,0,131,1,0,130,1,0, + 124,2,0,106,8,0,100,4,0,131,1,0,125,4,0,124, + 4,0,100,11,0,107,7,0,114,153,0,116,6,0,100,7, + 0,106,7,0,124,2,0,131,1,0,131,1,0,130,1,0, + 110,125,0,124,4,0,100,6,0,107,2,0,114,22,1,124, + 2,0,106,9,0,100,4,0,100,5,0,131,2,0,100,12, + 0,25,125,5,0,124,5,0,106,10,0,116,11,0,131,1, + 0,115,223,0,116,6,0,100,8,0,106,7,0,116,11,0, + 131,1,0,131,1,0,130,1,0,124,5,0,116,12,0,116, + 11,0,131,1,0,100,1,0,133,2,0,25,125,6,0,124, + 6,0,106,13,0,131,0,0,115,22,1,116,6,0,100,9, + 0,106,7,0,124,5,0,131,1,0,131,1,0,130,1,0, + 124,2,0,106,14,0,100,4,0,131,1,0,100,10,0,25, + 125,7,0,116,15,0,124,1,0,124,7,0,116,16,0,100, + 10,0,25,23,131,2,0,83,41,13,97,110,1,0,0,71, + 105,118,101,110,32,116,104,101,32,112,97,116,104,32,116,111, + 32,97,32,46,112,121,99,46,32,102,105,108,101,44,32,114, + 101,116,117,114,110,32,116,104,101,32,112,97,116,104,32,116, + 111,32,105,116,115,32,46,112,121,32,102,105,108,101,46,10, + 10,32,32,32,32,84,104,101,32,46,112,121,99,32,102,105, + 108,101,32,100,111,101,115,32,110,111,116,32,110,101,101,100, + 32,116,111,32,101,120,105,115,116,59,32,116,104,105,115,32, + 115,105,109,112,108,121,32,114,101,116,117,114,110,115,32,116, + 104,101,32,112,97,116,104,32,116,111,10,32,32,32,32,116, + 104,101,32,46,112,121,32,102,105,108,101,32,99,97,108,99, + 117,108,97,116,101,100,32,116,111,32,99,111,114,114,101,115, + 112,111,110,100,32,116,111,32,116,104,101,32,46,112,121,99, + 32,102,105,108,101,46,32,32,73,102,32,112,97,116,104,32, + 100,111,101,115,10,32,32,32,32,110,111,116,32,99,111,110, + 102,111,114,109,32,116,111,32,80,69,80,32,51,49,52,55, + 47,52,56,56,32,102,111,114,109,97,116,44,32,86,97,108, + 117,101,69,114,114,111,114,32,119,105,108,108,32,98,101,32, + 114,97,105,115,101,100,46,32,73,102,10,32,32,32,32,115, + 121,115,46,105,109,112,108,101,109,101,110,116,97,116,105,111, + 110,46,99,97,99,104,101,95,116,97,103,32,105,115,32,78, + 111,110,101,32,116,104,101,110,32,78,111,116,73,109,112,108, + 101,109,101,110,116,101,100,69,114,114,111,114,32,105,115,32, + 114,97,105,115,101,100,46,10,10,32,32,32,32,78,122,36, + 115,121,115,46,105,109,112,108,101,109,101,110,116,97,116,105, + 111,110,46,99,97,99,104,101,95,116,97,103,32,105,115,32, + 78,111,110,101,122,37,123,125,32,110,111,116,32,98,111,116, + 116,111,109,45,108,101,118,101,108,32,100,105,114,101,99,116, + 111,114,121,32,105,110,32,123,33,114,125,114,58,0,0,0, + 114,56,0,0,0,233,3,0,0,0,122,33,101,120,112,101, + 99,116,101,100,32,111,110,108,121,32,50,32,111,114,32,51, + 32,100,111,116,115,32,105,110,32,123,33,114,125,122,57,111, + 112,116,105,109,105,122,97,116,105,111,110,32,112,111,114,116, + 105,111,110,32,111,102,32,102,105,108,101,110,97,109,101,32, + 100,111,101,115,32,110,111,116,32,115,116,97,114,116,32,119, + 105,116,104,32,123,33,114,125,122,52,111,112,116,105,109,105, + 122,97,116,105,111,110,32,108,101,118,101,108,32,123,33,114, + 125,32,105,115,32,110,111,116,32,97,110,32,97,108,112,104, + 97,110,117,109,101,114,105,99,32,118,97,108,117,101,114,59, + 0,0,0,62,2,0,0,0,114,56,0,0,0,114,80,0, + 0,0,233,254,255,255,255,41,17,114,7,0,0,0,114,64, + 0,0,0,114,65,0,0,0,114,66,0,0,0,114,38,0, + 0,0,114,73,0,0,0,114,71,0,0,0,114,47,0,0, + 0,218,5,99,111,117,110,116,114,34,0,0,0,114,9,0, + 0,0,114,72,0,0,0,114,31,0,0,0,114,70,0,0, + 0,218,9,112,97,114,116,105,116,105,111,110,114,28,0,0, + 0,218,15,83,79,85,82,67,69,95,83,85,70,70,73,88, + 69,83,41,8,114,35,0,0,0,114,76,0,0,0,90,16, + 112,121,99,97,99,104,101,95,102,105,108,101,110,97,109,101, + 90,7,112,121,99,97,99,104,101,90,9,100,111,116,95,99, + 111,117,110,116,114,57,0,0,0,90,9,111,112,116,95,108, + 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, + 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, + 99,97,99,104,101,31,1,0,0,115,44,0,0,0,0,9, + 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, + 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, + 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, + 1,0,0,0,0,0,0,0,5,0,0,0,12,0,0,0, + 67,0,0,0,115,164,0,0,0,116,0,0,124,0,0,131, + 1,0,100,1,0,107,2,0,114,22,0,100,2,0,83,124, + 0,0,106,1,0,100,3,0,131,1,0,92,3,0,125,1, + 0,125,2,0,125,3,0,124,1,0,12,115,81,0,124,3, + 0,106,2,0,131,0,0,100,7,0,100,8,0,133,2,0, + 25,100,6,0,107,3,0,114,85,0,124,0,0,83,121,16, + 0,116,3,0,124,0,0,131,1,0,125,4,0,87,110,40, + 0,4,116,4,0,116,5,0,102,2,0,107,10,0,114,143, + 0,1,1,1,124,0,0,100,2,0,100,9,0,133,2,0, + 25,125,4,0,89,110,1,0,88,116,6,0,124,4,0,131, + 1,0,114,160,0,124,4,0,83,124,0,0,83,41,10,122, + 188,67,111,110,118,101,114,116,32,97,32,98,121,116,101,99, + 111,100,101,32,102,105,108,101,32,112,97,116,104,32,116,111, + 32,97,32,115,111,117,114,99,101,32,112,97,116,104,32,40, + 105,102,32,112,111,115,115,105,98,108,101,41,46,10,10,32, + 32,32,32,84,104,105,115,32,102,117,110,99,116,105,111,110, + 32,101,120,105,115,116,115,32,112,117,114,101,108,121,32,102, + 111,114,32,98,97,99,107,119,97,114,100,115,45,99,111,109, + 112,97,116,105,98,105,108,105,116,121,32,102,111,114,10,32, + 32,32,32,80,121,73,109,112,111,114,116,95,69,120,101,99, + 67,111,100,101,77,111,100,117,108,101,87,105,116,104,70,105, + 108,101,110,97,109,101,115,40,41,32,105,110,32,116,104,101, + 32,67,32,65,80,73,46,10,10,32,32,32,32,114,59,0, + 0,0,78,114,58,0,0,0,114,80,0,0,0,114,29,0, + 0,0,90,2,112,121,233,253,255,255,255,233,255,255,255,255, + 114,87,0,0,0,41,7,114,31,0,0,0,114,32,0,0, + 0,218,5,108,111,119,101,114,114,85,0,0,0,114,66,0, + 0,0,114,71,0,0,0,114,44,0,0,0,41,5,218,13, + 98,121,116,101,99,111,100,101,95,112,97,116,104,114,78,0, + 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, + 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,64, + 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, + 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, + 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, + 0,0,67,0,0,0,115,92,0,0,0,124,0,0,106,0, + 0,116,1,0,116,2,0,131,1,0,131,1,0,114,59,0, + 121,14,0,116,3,0,124,0,0,131,1,0,83,87,113,88, + 0,4,116,4,0,107,10,0,114,55,0,1,1,1,89,113, + 88,0,88,110,29,0,124,0,0,106,0,0,116,1,0,116, + 5,0,131,1,0,131,1,0,114,84,0,124,0,0,83,100, + 0,0,83,100,0,0,83,41,1,78,41,6,218,8,101,110, + 100,115,119,105,116,104,218,5,116,117,112,108,101,114,84,0, + 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, + 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, + 101,116,95,99,97,99,104,101,100,83,1,0,0,115,16,0, + 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, + 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, + 0,121,19,0,116,0,0,124,0,0,131,1,0,106,1,0, + 125,1,0,87,110,24,0,4,116,2,0,107,10,0,114,45, + 0,1,1,1,100,1,0,125,1,0,89,110,1,0,88,124, + 1,0,100,2,0,79,125,1,0,124,1,0,83,41,3,122, + 51,67,97,108,99,117,108,97,116,101,32,116,104,101,32,109, + 111,100,101,32,112,101,114,109,105,115,115,105,111,110,115,32, + 102,111,114,32,97,32,98,121,116,101,99,111,100,101,32,102, + 105,108,101,46,105,182,1,0,0,233,128,0,0,0,41,3, + 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, + 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, + 99,95,109,111,100,101,95,1,0,0,115,12,0,0,0,0, + 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, + 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, + 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, + 102,1,0,100,2,0,100,3,0,134,1,0,125,1,0,121, + 13,0,116,0,0,106,1,0,125,2,0,87,110,30,0,4, + 116,2,0,107,10,0,114,66,0,1,1,1,100,4,0,100, + 5,0,132,0,0,125,2,0,89,110,1,0,88,124,2,0, + 124,1,0,136,0,0,131,2,0,1,124,1,0,83,41,6, + 122,252,68,101,99,111,114,97,116,111,114,32,116,111,32,118, + 101,114,105,102,121,32,116,104,97,116,32,116,104,101,32,109, + 111,100,117,108,101,32,98,101,105,110,103,32,114,101,113,117, + 101,115,116,101,100,32,109,97,116,99,104,101,115,32,116,104, + 101,32,111,110,101,32,116,104,101,10,32,32,32,32,108,111, + 97,100,101,114,32,99,97,110,32,104,97,110,100,108,101,46, + 10,10,32,32,32,32,84,104,101,32,102,105,114,115,116,32, + 97,114,103,117,109,101,110,116,32,40,115,101,108,102,41,32, + 109,117,115,116,32,100,101,102,105,110,101,32,95,110,97,109, + 101,32,119,104,105,99,104,32,116,104,101,32,115,101,99,111, + 110,100,32,97,114,103,117,109,101,110,116,32,105,115,10,32, + 32,32,32,99,111,109,112,97,114,101,100,32,97,103,97,105, + 110,115,116,46,32,73,102,32,116,104,101,32,99,111,109,112, + 97,114,105,115,111,110,32,102,97,105,108,115,32,116,104,101, + 110,32,73,109,112,111,114,116,69,114,114,111,114,32,105,115, + 32,114,97,105,115,101,100,46,10,10,32,32,32,32,78,99, + 2,0,0,0,0,0,0,0,4,0,0,0,5,0,0,0, + 31,0,0,0,115,89,0,0,0,124,1,0,100,0,0,107, + 8,0,114,24,0,124,0,0,106,0,0,125,1,0,110,46, + 0,124,0,0,106,0,0,124,1,0,107,3,0,114,70,0, + 116,1,0,100,1,0,124,0,0,106,0,0,124,1,0,102, + 2,0,22,100,2,0,124,1,0,131,1,1,130,1,0,136, + 0,0,124,0,0,124,1,0,124,2,0,124,3,0,142,2, + 0,83,41,3,78,122,30,108,111,97,100,101,114,32,102,111, + 114,32,37,115,32,99,97,110,110,111,116,32,104,97,110,100, + 108,101,32,37,115,218,4,110,97,109,101,41,2,114,98,0, + 0,0,218,11,73,109,112,111,114,116,69,114,114,111,114,41, + 4,218,4,115,101,108,102,114,98,0,0,0,218,4,97,114, + 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,123,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,115,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -595,17 +573,17 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,134,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,126,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, - 98,111,111,116,115,116,114,97,112,114,120,0,0,0,218,9, - 78,97,109,101,69,114,114,111,114,41,3,114,109,0,0,0, - 114,110,0,0,0,114,120,0,0,0,114,4,0,0,0,41, - 1,114,109,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,115,1,0,0,115,14,0,0, + 98,111,111,116,115,116,114,97,112,114,113,0,0,0,218,9, + 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, + 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, + 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, + 101,99,107,95,110,97,109,101,107,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, - 123,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, + 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, 0,106,0,0,124,1,0,131,1,0,92,2,0,125,2,0, 125,3,0,124,2,0,100,1,0,107,8,0,114,80,0,116, @@ -628,14 +606,14 @@ const unsigned char _Py_M__importlib_external[] = { 114,59,0,0,0,41,6,218,11,102,105,110,100,95,108,111, 97,100,101,114,114,31,0,0,0,114,60,0,0,0,114,61, 0,0,0,114,47,0,0,0,218,13,73,109,112,111,114,116, - 87,97,114,110,105,110,103,41,5,114,108,0,0,0,218,8, + 87,97,114,110,105,110,103,41,5,114,100,0,0,0,218,8, 102,117,108,108,110,97,109,101,218,6,108,111,97,100,101,114, 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,143,1,0,0,115,10,0,0,0,0,10,21,1,24,1, - 6,1,29,1,114,130,0,0,0,99,4,0,0,0,0,0, - 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,228, + 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, + 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, 9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110, 6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9, @@ -643,1954 +621,1959 @@ const unsigned char _Py_M__importlib_external[] = { 0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0, 0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0, 0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5, - 0,116,0,0,107,3,0,114,165,0,100,8,0,106,1,0, - 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,124, - 8,0,131,1,0,1,116,3,0,124,8,0,124,4,0,141, - 1,0,130,1,0,110,113,0,116,4,0,124,6,0,131,1, - 0,100,5,0,107,3,0,114,223,0,100,9,0,106,1,0, - 124,2,0,131,1,0,125,8,0,116,2,0,124,8,0,131, - 1,0,1,116,5,0,124,8,0,131,1,0,130,1,0,110, - 55,0,116,4,0,124,7,0,131,1,0,100,5,0,107,3, - 0,114,22,1,100,10,0,106,1,0,124,2,0,131,1,0, - 125,8,0,116,2,0,124,8,0,131,1,0,1,116,5,0, - 124,8,0,131,1,0,130,1,0,124,1,0,100,1,0,107, - 9,0,114,214,1,121,20,0,116,6,0,124,1,0,100,11, - 0,25,131,1,0,125,9,0,87,110,18,0,4,116,7,0, - 107,10,0,114,74,1,1,1,1,89,110,59,0,88,116,8, - 0,124,6,0,131,1,0,124,9,0,107,3,0,114,133,1, - 100,12,0,106,1,0,124,2,0,131,1,0,125,8,0,116, - 2,0,124,8,0,131,1,0,1,116,3,0,124,8,0,124, - 4,0,141,1,0,130,1,0,121,18,0,124,1,0,100,13, - 0,25,100,14,0,64,125,10,0,87,110,18,0,4,116,7, - 0,107,10,0,114,171,1,1,1,1,89,110,43,0,88,116, - 8,0,124,7,0,131,1,0,124,10,0,107,3,0,114,214, - 1,116,3,0,100,12,0,106,1,0,124,2,0,131,1,0, - 124,4,0,141,1,0,130,1,0,124,0,0,100,7,0,100, - 1,0,133,2,0,25,83,41,15,97,122,1,0,0,86,97, - 108,105,100,97,116,101,32,116,104,101,32,104,101,97,100,101, - 114,32,111,102,32,116,104,101,32,112,97,115,115,101,100,45, - 105,110,32,98,121,116,101,99,111,100,101,32,97,103,97,105, - 110,115,116,32,115,111,117,114,99,101,95,115,116,97,116,115, - 32,40,105,102,10,32,32,32,32,103,105,118,101,110,41,32, - 97,110,100,32,114,101,116,117,114,110,105,110,103,32,116,104, - 101,32,98,121,116,101,99,111,100,101,32,116,104,97,116,32, - 99,97,110,32,98,101,32,99,111,109,112,105,108,101,100,32, - 98,121,32,99,111,109,112,105,108,101,40,41,46,10,10,32, - 32,32,32,65,108,108,32,111,116,104,101,114,32,97,114,103, - 117,109,101,110,116,115,32,97,114,101,32,117,115,101,100,32, - 116,111,32,101,110,104,97,110,99,101,32,101,114,114,111,114, - 32,114,101,112,111,114,116,105,110,103,46,10,10,32,32,32, - 32,73,109,112,111,114,116,69,114,114,111,114,32,105,115,32, - 114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32, - 109,97,103,105,99,32,110,117,109,98,101,114,32,105,115,32, - 105,110,99,111,114,114,101,99,116,32,111,114,32,116,104,101, - 32,98,121,116,101,99,111,100,101,32,105,115,10,32,32,32, - 32,102,111,117,110,100,32,116,111,32,98,101,32,115,116,97, - 108,101,46,32,69,79,70,69,114,114,111,114,32,105,115,32, - 114,97,105,115,101,100,32,119,104,101,110,32,116,104,101,32, - 100,97,116,97,32,105,115,32,102,111,117,110,100,32,116,111, - 32,98,101,10,32,32,32,32,116,114,117,110,99,97,116,101, - 100,46,10,10,32,32,32,32,78,114,106,0,0,0,122,10, - 60,98,121,116,101,99,111,100,101,62,114,35,0,0,0,114, - 12,0,0,0,233,8,0,0,0,233,12,0,0,0,122,30, - 98,97,100,32,109,97,103,105,99,32,110,117,109,98,101,114, - 32,105,110,32,123,33,114,125,58,32,123,33,114,125,122,43, - 114,101,97,99,104,101,100,32,69,79,70,32,119,104,105,108, - 101,32,114,101,97,100,105,110,103,32,116,105,109,101,115,116, - 97,109,112,32,105,110,32,123,33,114,125,122,48,114,101,97, - 99,104,101,100,32,69,79,70,32,119,104,105,108,101,32,114, - 101,97,100,105,110,103,32,115,105,122,101,32,111,102,32,115, - 111,117,114,99,101,32,105,110,32,123,33,114,125,218,5,109, - 116,105,109,101,122,26,98,121,116,101,99,111,100,101,32,105, - 115,32,115,116,97,108,101,32,102,111,114,32,123,33,114,125, - 218,4,115,105,122,101,108,3,0,0,0,255,127,255,127,3, - 0,41,9,218,12,77,65,71,73,67,95,78,85,77,66,69, - 82,114,47,0,0,0,114,105,0,0,0,114,107,0,0,0, - 114,31,0,0,0,218,8,69,79,70,69,114,114,111,114,114, - 14,0,0,0,218,8,75,101,121,69,114,114,111,114,114,19, - 0,0,0,41,11,114,53,0,0,0,218,12,115,111,117,114, - 99,101,95,115,116,97,116,115,114,106,0,0,0,114,35,0, - 0,0,90,11,101,120,99,95,100,101,116,97,105,108,115,90, - 5,109,97,103,105,99,90,13,114,97,119,95,116,105,109,101, - 115,116,97,109,112,90,8,114,97,119,95,115,105,122,101,114, - 75,0,0,0,218,12,115,111,117,114,99,101,95,109,116,105, - 109,101,218,11,115,111,117,114,99,101,95,115,105,122,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,25, - 95,118,97,108,105,100,97,116,101,95,98,121,116,101,99,111, - 100,101,95,104,101,97,100,101,114,160,1,0,0,115,76,0, - 0,0,0,11,6,1,12,1,13,3,6,1,12,1,10,1, - 16,1,16,1,16,1,12,1,18,1,10,1,18,1,18,1, - 15,1,10,1,15,1,18,1,15,1,10,1,12,1,12,1, - 3,1,20,1,13,1,5,2,18,1,15,1,10,1,15,1, - 3,1,18,1,13,1,5,2,18,1,15,1,9,1,114,141, - 0,0,0,99,4,0,0,0,0,0,0,0,5,0,0,0, - 6,0,0,0,67,0,0,0,115,112,0,0,0,116,0,0, - 106,1,0,124,0,0,131,1,0,125,4,0,116,2,0,124, - 4,0,116,3,0,131,2,0,114,75,0,116,4,0,100,1, - 0,124,2,0,131,2,0,1,124,3,0,100,2,0,107,9, - 0,114,71,0,116,5,0,106,6,0,124,4,0,124,3,0, - 131,2,0,1,124,4,0,83,116,7,0,100,3,0,106,8, - 0,124,2,0,131,1,0,100,4,0,124,1,0,100,5,0, - 124,2,0,131,1,2,130,1,0,100,2,0,83,41,6,122, - 60,67,111,109,112,105,108,101,32,98,121,116,101,99,111,100, - 101,32,97,115,32,114,101,116,117,114,110,101,100,32,98,121, - 32,95,118,97,108,105,100,97,116,101,95,98,121,116,101,99, - 111,100,101,95,104,101,97,100,101,114,40,41,46,122,21,99, - 111,100,101,32,111,98,106,101,99,116,32,102,114,111,109,32, - 123,33,114,125,78,122,23,78,111,110,45,99,111,100,101,32, - 111,98,106,101,99,116,32,105,110,32,123,33,114,125,114,106, - 0,0,0,114,35,0,0,0,41,9,218,7,109,97,114,115, - 104,97,108,90,5,108,111,97,100,115,218,10,105,115,105,110, - 115,116,97,110,99,101,218,10,95,99,111,100,101,95,116,121, - 112,101,114,105,0,0,0,218,4,95,105,109,112,90,16,95, - 102,105,120,95,99,111,95,102,105,108,101,110,97,109,101,114, - 107,0,0,0,114,47,0,0,0,41,5,114,53,0,0,0, - 114,106,0,0,0,114,89,0,0,0,114,90,0,0,0,218, - 4,99,111,100,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,95,99,111,109,112,105,108,101,95,98, - 121,116,101,99,111,100,101,215,1,0,0,115,16,0,0,0, - 0,2,15,1,15,1,13,1,12,1,16,1,4,2,18,1, - 114,147,0,0,0,114,59,0,0,0,99,3,0,0,0,0, - 0,0,0,4,0,0,0,3,0,0,0,67,0,0,0,115, - 76,0,0,0,116,0,0,116,1,0,131,1,0,125,3,0, - 124,3,0,106,2,0,116,3,0,124,1,0,131,1,0,131, - 1,0,1,124,3,0,106,2,0,116,3,0,124,2,0,131, - 1,0,131,1,0,1,124,3,0,106,2,0,116,4,0,106, - 5,0,124,0,0,131,1,0,131,1,0,1,124,3,0,83, - 41,1,122,80,67,111,109,112,105,108,101,32,97,32,99,111, - 100,101,32,111,98,106,101,99,116,32,105,110,116,111,32,98, - 121,116,101,99,111,100,101,32,102,111,114,32,119,114,105,116, - 105,110,103,32,111,117,116,32,116,111,32,97,32,98,121,116, - 101,45,99,111,109,112,105,108,101,100,10,32,32,32,32,102, - 105,108,101,46,41,6,218,9,98,121,116,101,97,114,114,97, - 121,114,135,0,0,0,218,6,101,120,116,101,110,100,114,17, - 0,0,0,114,142,0,0,0,90,5,100,117,109,112,115,41, - 4,114,146,0,0,0,114,133,0,0,0,114,140,0,0,0, - 114,53,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,17,95,99,111,100,101,95,116,111,95,98, - 121,116,101,99,111,100,101,227,1,0,0,115,10,0,0,0, - 0,3,12,1,19,1,19,1,22,1,114,150,0,0,0,99, - 1,0,0,0,0,0,0,0,5,0,0,0,4,0,0,0, - 67,0,0,0,115,89,0,0,0,100,1,0,100,2,0,108, - 0,0,125,1,0,116,1,0,106,2,0,124,0,0,131,1, - 0,106,3,0,125,2,0,124,1,0,106,4,0,124,2,0, - 131,1,0,125,3,0,116,1,0,106,5,0,100,2,0,100, - 3,0,131,2,0,125,4,0,124,4,0,106,6,0,124,0, - 0,106,6,0,124,3,0,100,1,0,25,131,1,0,131,1, - 0,83,41,4,122,121,68,101,99,111,100,101,32,98,121,116, - 101,115,32,114,101,112,114,101,115,101,110,116,105,110,103,32, - 115,111,117,114,99,101,32,99,111,100,101,32,97,110,100,32, - 114,101,116,117,114,110,32,116,104,101,32,115,116,114,105,110, - 103,46,10,10,32,32,32,32,85,110,105,118,101,114,115,97, - 108,32,110,101,119,108,105,110,101,32,115,117,112,112,111,114, - 116,32,105,115,32,117,115,101,100,32,105,110,32,116,104,101, - 32,100,101,99,111,100,105,110,103,46,10,32,32,32,32,114, - 59,0,0,0,78,84,41,7,218,8,116,111,107,101,110,105, - 122,101,114,49,0,0,0,90,7,66,121,116,101,115,73,79, - 90,8,114,101,97,100,108,105,110,101,90,15,100,101,116,101, - 99,116,95,101,110,99,111,100,105,110,103,90,25,73,110,99, - 114,101,109,101,110,116,97,108,78,101,119,108,105,110,101,68, - 101,99,111,100,101,114,218,6,100,101,99,111,100,101,41,5, - 218,12,115,111,117,114,99,101,95,98,121,116,101,115,114,151, - 0,0,0,90,21,115,111,117,114,99,101,95,98,121,116,101, - 115,95,114,101,97,100,108,105,110,101,218,8,101,110,99,111, - 100,105,110,103,90,15,110,101,119,108,105,110,101,95,100,101, - 99,111,100,101,114,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,13,100,101,99,111,100,101,95,115,111,117, - 114,99,101,237,1,0,0,115,10,0,0,0,0,5,12,1, - 18,1,15,1,18,1,114,155,0,0,0,114,127,0,0,0, - 218,26,115,117,98,109,111,100,117,108,101,95,115,101,97,114, - 99,104,95,108,111,99,97,116,105,111,110,115,99,2,0,0, - 0,2,0,0,0,9,0,0,0,19,0,0,0,67,0,0, - 0,115,89,1,0,0,124,1,0,100,1,0,107,8,0,114, - 73,0,100,2,0,125,1,0,116,0,0,124,2,0,100,3, - 0,131,2,0,114,73,0,121,19,0,124,2,0,106,1,0, - 124,0,0,131,1,0,125,1,0,87,110,18,0,4,116,2, - 0,107,10,0,114,72,0,1,1,1,89,110,1,0,88,116, - 3,0,106,4,0,124,0,0,124,2,0,100,4,0,124,1, - 0,131,2,1,125,4,0,100,5,0,124,4,0,95,5,0, - 124,2,0,100,1,0,107,8,0,114,194,0,120,73,0,116, - 6,0,131,0,0,68,93,58,0,92,2,0,125,5,0,125, - 6,0,124,1,0,106,7,0,116,8,0,124,6,0,131,1, - 0,131,1,0,114,128,0,124,5,0,124,0,0,124,1,0, - 131,2,0,125,2,0,124,2,0,124,4,0,95,9,0,80, - 113,128,0,87,100,1,0,83,124,3,0,116,10,0,107,8, - 0,114,23,1,116,0,0,124,2,0,100,6,0,131,2,0, - 114,32,1,121,19,0,124,2,0,106,11,0,124,0,0,131, - 1,0,125,7,0,87,110,18,0,4,116,2,0,107,10,0, - 114,4,1,1,1,1,89,113,32,1,88,124,7,0,114,32, - 1,103,0,0,124,4,0,95,12,0,110,9,0,124,3,0, - 124,4,0,95,12,0,124,4,0,106,12,0,103,0,0,107, - 2,0,114,85,1,124,1,0,114,85,1,116,13,0,124,1, - 0,131,1,0,100,7,0,25,125,8,0,124,4,0,106,12, - 0,106,14,0,124,8,0,131,1,0,1,124,4,0,83,41, - 8,97,61,1,0,0,82,101,116,117,114,110,32,97,32,109, - 111,100,117,108,101,32,115,112,101,99,32,98,97,115,101,100, - 32,111,110,32,97,32,102,105,108,101,32,108,111,99,97,116, - 105,111,110,46,10,10,32,32,32,32,84,111,32,105,110,100, - 105,99,97,116,101,32,116,104,97,116,32,116,104,101,32,109, - 111,100,117,108,101,32,105,115,32,97,32,112,97,99,107,97, - 103,101,44,32,115,101,116,10,32,32,32,32,115,117,98,109, - 111,100,117,108,101,95,115,101,97,114,99,104,95,108,111,99, - 97,116,105,111,110,115,32,116,111,32,97,32,108,105,115,116, - 32,111,102,32,100,105,114,101,99,116,111,114,121,32,112,97, - 116,104,115,46,32,32,65,110,10,32,32,32,32,101,109,112, - 116,121,32,108,105,115,116,32,105,115,32,115,117,102,102,105, - 99,105,101,110,116,44,32,116,104,111,117,103,104,32,105,116, - 115,32,110,111,116,32,111,116,104,101,114,119,105,115,101,32, - 117,115,101,102,117,108,32,116,111,32,116,104,101,10,32,32, - 32,32,105,109,112,111,114,116,32,115,121,115,116,101,109,46, - 10,10,32,32,32,32,84,104,101,32,108,111,97,100,101,114, - 32,109,117,115,116,32,116,97,107,101,32,97,32,115,112,101, - 99,32,97,115,32,105,116,115,32,111,110,108,121,32,95,95, - 105,110,105,116,95,95,40,41,32,97,114,103,46,10,10,32, - 32,32,32,78,122,9,60,117,110,107,110,111,119,110,62,218, - 12,103,101,116,95,102,105,108,101,110,97,109,101,218,6,111, - 114,105,103,105,110,84,218,10,105,115,95,112,97,99,107,97, - 103,101,114,59,0,0,0,41,15,114,115,0,0,0,114,157, - 0,0,0,114,107,0,0,0,114,121,0,0,0,218,10,77, - 111,100,117,108,101,83,112,101,99,90,13,95,115,101,116,95, - 102,105,108,101,97,116,116,114,218,27,95,103,101,116,95,115, - 117,112,112,111,114,116,101,100,95,102,105,108,101,95,108,111, - 97,100,101,114,115,114,92,0,0,0,114,93,0,0,0,114, - 127,0,0,0,218,9,95,80,79,80,85,76,65,84,69,114, - 159,0,0,0,114,156,0,0,0,114,38,0,0,0,218,6, - 97,112,112,101,110,100,41,9,114,106,0,0,0,90,8,108, - 111,99,97,116,105,111,110,114,127,0,0,0,114,156,0,0, - 0,218,4,115,112,101,99,218,12,108,111,97,100,101,114,95, - 99,108,97,115,115,218,8,115,117,102,102,105,120,101,115,114, - 159,0,0,0,90,7,100,105,114,110,97,109,101,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,23,115,112, - 101,99,95,102,114,111,109,95,102,105,108,101,95,108,111,99, - 97,116,105,111,110,254,1,0,0,115,60,0,0,0,0,12, - 12,4,6,1,15,2,3,1,19,1,13,1,5,8,24,1, - 9,3,12,1,22,1,21,1,15,1,9,1,5,2,4,3, - 12,2,15,1,3,1,19,1,13,1,5,2,6,1,12,2, - 9,1,15,1,6,1,16,1,16,2,114,167,0,0,0,99, - 0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0, - 64,0,0,0,115,121,0,0,0,101,0,0,90,1,0,100, - 0,0,90,2,0,100,1,0,90,3,0,100,2,0,90,4, - 0,100,3,0,90,5,0,100,4,0,90,6,0,101,7,0, - 100,5,0,100,6,0,132,0,0,131,1,0,90,8,0,101, - 7,0,100,7,0,100,8,0,132,0,0,131,1,0,90,9, - 0,101,7,0,100,9,0,100,9,0,100,10,0,100,11,0, - 132,2,0,131,1,0,90,10,0,101,7,0,100,9,0,100, - 12,0,100,13,0,132,1,0,131,1,0,90,11,0,100,9, - 0,83,41,14,218,21,87,105,110,100,111,119,115,82,101,103, - 105,115,116,114,121,70,105,110,100,101,114,122,62,77,101,116, - 97,32,112,97,116,104,32,102,105,110,100,101,114,32,102,111, - 114,32,109,111,100,117,108,101,115,32,100,101,99,108,97,114, - 101,100,32,105,110,32,116,104,101,32,87,105,110,100,111,119, - 115,32,114,101,103,105,115,116,114,121,46,122,59,83,111,102, - 116,119,97,114,101,92,80,121,116,104,111,110,92,80,121,116, - 104,111,110,67,111,114,101,92,123,115,121,115,95,118,101,114, - 115,105,111,110,125,92,77,111,100,117,108,101,115,92,123,102, - 117,108,108,110,97,109,101,125,122,65,83,111,102,116,119,97, - 114,101,92,80,121,116,104,111,110,92,80,121,116,104,111,110, - 67,111,114,101,92,123,115,121,115,95,118,101,114,115,105,111, - 110,125,92,77,111,100,117,108,101,115,92,123,102,117,108,108, - 110,97,109,101,125,92,68,101,98,117,103,70,99,2,0,0, - 0,0,0,0,0,2,0,0,0,11,0,0,0,67,0,0, - 0,115,67,0,0,0,121,23,0,116,0,0,106,1,0,116, - 0,0,106,2,0,124,1,0,131,2,0,83,87,110,37,0, - 4,116,3,0,107,10,0,114,62,0,1,1,1,116,0,0, - 106,1,0,116,0,0,106,4,0,124,1,0,131,2,0,83, - 89,110,1,0,88,100,0,0,83,41,1,78,41,5,218,7, - 95,119,105,110,114,101,103,90,7,79,112,101,110,75,101,121, - 90,17,72,75,69,89,95,67,85,82,82,69,78,84,95,85, - 83,69,82,114,40,0,0,0,90,18,72,75,69,89,95,76, - 79,67,65,76,95,77,65,67,72,73,78,69,41,2,218,3, - 99,108,115,218,3,107,101,121,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,14,95,111,112,101,110,95,114, - 101,103,105,115,116,114,121,76,2,0,0,115,8,0,0,0, - 0,2,3,1,23,1,13,1,122,36,87,105,110,100,111,119, - 115,82,101,103,105,115,116,114,121,70,105,110,100,101,114,46, - 95,111,112,101,110,95,114,101,103,105,115,116,114,121,99,2, - 0,0,0,0,0,0,0,6,0,0,0,16,0,0,0,67, - 0,0,0,115,143,0,0,0,124,0,0,106,0,0,114,21, - 0,124,0,0,106,1,0,125,2,0,110,9,0,124,0,0, - 106,2,0,125,2,0,124,2,0,106,3,0,100,1,0,124, - 1,0,100,2,0,116,4,0,106,5,0,100,0,0,100,3, - 0,133,2,0,25,131,0,2,125,3,0,121,47,0,124,0, - 0,106,6,0,124,3,0,131,1,0,143,25,0,125,4,0, - 116,7,0,106,8,0,124,4,0,100,4,0,131,2,0,125, - 5,0,87,100,0,0,81,82,88,87,110,22,0,4,116,9, - 0,107,10,0,114,138,0,1,1,1,100,0,0,83,89,110, - 1,0,88,124,5,0,83,41,5,78,114,126,0,0,0,90, - 11,115,121,115,95,118,101,114,115,105,111,110,114,80,0,0, - 0,114,30,0,0,0,41,10,218,11,68,69,66,85,71,95, - 66,85,73,76,68,218,18,82,69,71,73,83,84,82,89,95, - 75,69,89,95,68,69,66,85,71,218,12,82,69,71,73,83, - 84,82,89,95,75,69,89,114,47,0,0,0,114,7,0,0, - 0,218,7,118,101,114,115,105,111,110,114,172,0,0,0,114, - 169,0,0,0,90,10,81,117,101,114,121,86,97,108,117,101, - 114,40,0,0,0,41,6,114,170,0,0,0,114,126,0,0, - 0,90,12,114,101,103,105,115,116,114,121,95,107,101,121,114, - 171,0,0,0,90,4,104,107,101,121,218,8,102,105,108,101, - 112,97,116,104,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,16,95,115,101,97,114,99,104,95,114,101,103, - 105,115,116,114,121,83,2,0,0,115,22,0,0,0,0,2, - 9,1,12,2,9,1,15,1,22,1,3,1,18,1,29,1, - 13,1,9,1,122,38,87,105,110,100,111,119,115,82,101,103, - 105,115,116,114,121,70,105,110,100,101,114,46,95,115,101,97, - 114,99,104,95,114,101,103,105,115,116,114,121,78,99,4,0, - 0,0,0,0,0,0,8,0,0,0,14,0,0,0,67,0, - 0,0,115,158,0,0,0,124,0,0,106,0,0,124,1,0, - 131,1,0,125,4,0,124,4,0,100,0,0,107,8,0,114, - 31,0,100,0,0,83,121,14,0,116,1,0,124,4,0,131, - 1,0,1,87,110,22,0,4,116,2,0,107,10,0,114,69, - 0,1,1,1,100,0,0,83,89,110,1,0,88,120,81,0, - 116,3,0,131,0,0,68,93,70,0,92,2,0,125,5,0, - 125,6,0,124,4,0,106,4,0,116,5,0,124,6,0,131, - 1,0,131,1,0,114,80,0,116,6,0,106,7,0,124,1, - 0,124,5,0,124,1,0,124,4,0,131,2,0,100,1,0, - 124,4,0,131,2,1,125,7,0,124,7,0,83,113,80,0, - 87,100,0,0,83,41,2,78,114,158,0,0,0,41,8,114, - 178,0,0,0,114,39,0,0,0,114,40,0,0,0,114,161, - 0,0,0,114,92,0,0,0,114,93,0,0,0,114,121,0, - 0,0,218,16,115,112,101,99,95,102,114,111,109,95,108,111, - 97,100,101,114,41,8,114,170,0,0,0,114,126,0,0,0, - 114,35,0,0,0,218,6,116,97,114,103,101,116,114,177,0, - 0,0,114,127,0,0,0,114,166,0,0,0,114,164,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,9,102,105,110,100,95,115,112,101,99,98,2,0,0,115, - 26,0,0,0,0,2,15,1,12,1,4,1,3,1,14,1, - 13,1,9,1,22,1,21,1,9,1,15,1,9,1,122,31, + 0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0, + 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,106, + 3,0,124,8,0,131,1,0,1,116,4,0,124,8,0,124, + 4,0,141,1,0,130,1,0,110,119,0,116,5,0,124,6, + 0,131,1,0,100,5,0,107,3,0,114,229,0,100,9,0, + 106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,106, + 3,0,124,8,0,131,1,0,1,116,6,0,124,8,0,131, + 1,0,130,1,0,110,58,0,116,5,0,124,7,0,131,1, + 0,100,5,0,107,3,0,114,31,1,100,10,0,106,1,0, + 124,2,0,131,1,0,125,8,0,116,2,0,106,3,0,124, + 8,0,131,1,0,1,116,6,0,124,8,0,131,1,0,130, + 1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20, + 0,116,7,0,124,1,0,100,11,0,25,131,1,0,125,9, + 0,87,110,18,0,4,116,8,0,107,10,0,114,83,1,1, + 1,1,89,110,62,0,88,116,9,0,124,6,0,131,1,0, + 124,9,0,107,3,0,114,145,1,100,12,0,106,1,0,124, + 2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,8, + 0,131,1,0,1,116,4,0,124,8,0,124,4,0,141,1, + 0,130,1,0,121,18,0,124,1,0,100,13,0,25,100,14, + 0,64,125,10,0,87,110,18,0,4,116,8,0,107,10,0, + 114,183,1,1,1,1,89,110,43,0,88,116,9,0,124,7, + 0,131,1,0,124,10,0,107,3,0,114,226,1,116,4,0, + 100,12,0,106,1,0,124,2,0,131,1,0,124,4,0,141, + 1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2, + 0,25,83,41,15,97,122,1,0,0,86,97,108,105,100,97, + 116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102, + 32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98, + 121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32, + 115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102, + 10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32, + 114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121, + 116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32, + 98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99, + 111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65, + 108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110, + 116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101, + 110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112, + 111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112, + 111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115, + 101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105, + 99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111, + 114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116, + 101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117, + 110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32, + 69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115, + 101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97, + 32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10, + 32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10, + 32,32,32,32,78,114,98,0,0,0,122,10,60,98,121,116, + 101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0, + 233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32, + 109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32, + 123,33,114,125,58,32,123,33,114,125,122,43,114,101,97,99, + 104,101,100,32,69,79,70,32,119,104,105,108,101,32,114,101, + 97,100,105,110,103,32,116,105,109,101,115,116,97,109,112,32, + 105,110,32,123,33,114,125,122,48,114,101,97,99,104,101,100, + 32,69,79,70,32,119,104,105,108,101,32,114,101,97,100,105, + 110,103,32,115,105,122,101,32,111,102,32,115,111,117,114,99, + 101,32,105,110,32,123,33,114,125,218,5,109,116,105,109,101, + 122,26,98,121,116,101,99,111,100,101,32,105,115,32,115,116, + 97,108,101,32,102,111,114,32,123,33,114,125,218,4,115,105, + 122,101,108,3,0,0,0,255,127,255,127,3,0,41,10,218, + 12,77,65,71,73,67,95,78,85,77,66,69,82,114,47,0, + 0,0,114,114,0,0,0,218,16,95,118,101,114,98,111,115, + 101,95,109,101,115,115,97,103,101,114,99,0,0,0,114,31, + 0,0,0,218,8,69,79,70,69,114,114,111,114,114,14,0, + 0,0,218,8,75,101,121,69,114,114,111,114,114,19,0,0, + 0,41,11,114,53,0,0,0,218,12,115,111,117,114,99,101, + 95,115,116,97,116,115,114,98,0,0,0,114,35,0,0,0, + 90,11,101,120,99,95,100,101,116,97,105,108,115,90,5,109, + 97,103,105,99,90,13,114,97,119,95,116,105,109,101,115,116, + 97,109,112,90,8,114,97,119,95,115,105,122,101,114,75,0, + 0,0,218,12,115,111,117,114,99,101,95,109,116,105,109,101, + 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, + 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, + 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, + 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, + 16,1,16,1,12,1,18,1,13,1,18,1,18,1,15,1, + 13,1,15,1,18,1,15,1,13,1,12,1,12,1,3,1, + 20,1,13,1,5,2,18,1,15,1,13,1,15,1,3,1, + 18,1,13,1,5,2,18,1,15,1,9,1,114,135,0,0, + 0,99,4,0,0,0,0,0,0,0,5,0,0,0,6,0, + 0,0,67,0,0,0,115,115,0,0,0,116,0,0,106,1, + 0,124,0,0,131,1,0,125,4,0,116,2,0,124,4,0, + 116,3,0,131,2,0,114,78,0,116,4,0,106,5,0,100, + 1,0,124,2,0,131,2,0,1,124,3,0,100,2,0,107, + 9,0,114,74,0,116,6,0,106,7,0,124,4,0,124,3, + 0,131,2,0,1,124,4,0,83,116,8,0,100,3,0,106, + 9,0,124,2,0,131,1,0,100,4,0,124,1,0,100,5, + 0,124,2,0,131,1,2,130,1,0,100,2,0,83,41,6, + 122,60,67,111,109,112,105,108,101,32,98,121,116,101,99,111, + 100,101,32,97,115,32,114,101,116,117,114,110,101,100,32,98, + 121,32,95,118,97,108,105,100,97,116,101,95,98,121,116,101, + 99,111,100,101,95,104,101,97,100,101,114,40,41,46,122,21, + 99,111,100,101,32,111,98,106,101,99,116,32,102,114,111,109, + 32,123,33,114,125,78,122,23,78,111,110,45,99,111,100,101, + 32,111,98,106,101,99,116,32,105,110,32,123,33,114,125,114, + 98,0,0,0,114,35,0,0,0,41,10,218,7,109,97,114, + 115,104,97,108,90,5,108,111,97,100,115,218,10,105,115,105, + 110,115,116,97,110,99,101,218,10,95,99,111,100,101,95,116, + 121,112,101,114,114,0,0,0,114,129,0,0,0,218,4,95, + 105,109,112,90,16,95,102,105,120,95,99,111,95,102,105,108, + 101,110,97,109,101,114,99,0,0,0,114,47,0,0,0,41, + 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, + 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, + 112,105,108,101,95,98,121,116,101,99,111,100,101,207,1,0, + 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, + 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, + 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, + 0,67,0,0,0,115,76,0,0,0,116,0,0,116,1,0, + 131,1,0,125,3,0,124,3,0,106,2,0,116,3,0,124, + 1,0,131,1,0,131,1,0,1,124,3,0,106,2,0,116, + 3,0,124,2,0,131,1,0,131,1,0,1,124,3,0,106, + 2,0,116,4,0,106,5,0,124,0,0,131,1,0,131,1, + 0,1,124,3,0,83,41,1,122,80,67,111,109,112,105,108, + 101,32,97,32,99,111,100,101,32,111,98,106,101,99,116,32, + 105,110,116,111,32,98,121,116,101,99,111,100,101,32,102,111, + 114,32,119,114,105,116,105,110,103,32,111,117,116,32,116,111, + 32,97,32,98,121,116,101,45,99,111,109,112,105,108,101,100, + 10,32,32,32,32,102,105,108,101,46,41,6,218,9,98,121, + 116,101,97,114,114,97,121,114,128,0,0,0,218,6,101,120, + 116,101,110,100,114,17,0,0,0,114,136,0,0,0,90,5, + 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, + 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, + 101,95,116,111,95,98,121,116,101,99,111,100,101,219,1,0, + 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, + 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, + 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, + 1,0,100,2,0,108,0,0,125,1,0,116,1,0,106,2, + 0,124,0,0,131,1,0,106,3,0,125,2,0,124,1,0, + 106,4,0,124,2,0,131,1,0,125,3,0,116,1,0,106, + 5,0,100,2,0,100,3,0,131,2,0,125,4,0,124,4, + 0,106,6,0,124,0,0,106,6,0,124,3,0,100,1,0, + 25,131,1,0,131,1,0,83,41,4,122,121,68,101,99,111, + 100,101,32,98,121,116,101,115,32,114,101,112,114,101,115,101, + 110,116,105,110,103,32,115,111,117,114,99,101,32,99,111,100, + 101,32,97,110,100,32,114,101,116,117,114,110,32,116,104,101, + 32,115,116,114,105,110,103,46,10,10,32,32,32,32,85,110, + 105,118,101,114,115,97,108,32,110,101,119,108,105,110,101,32, + 115,117,112,112,111,114,116,32,105,115,32,117,115,101,100,32, + 105,110,32,116,104,101,32,100,101,99,111,100,105,110,103,46, + 10,32,32,32,32,114,59,0,0,0,78,84,41,7,218,8, + 116,111,107,101,110,105,122,101,114,49,0,0,0,90,7,66, + 121,116,101,115,73,79,90,8,114,101,97,100,108,105,110,101, + 90,15,100,101,116,101,99,116,95,101,110,99,111,100,105,110, + 103,90,25,73,110,99,114,101,109,101,110,116,97,108,78,101, + 119,108,105,110,101,68,101,99,111,100,101,114,218,6,100,101, + 99,111,100,101,41,5,218,12,115,111,117,114,99,101,95,98, + 121,116,101,115,114,145,0,0,0,90,21,115,111,117,114,99, + 101,95,98,121,116,101,115,95,114,101,97,100,108,105,110,101, + 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, + 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, + 100,101,95,115,111,117,114,99,101,229,1,0,0,115,10,0, + 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, + 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, + 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, + 110,115,99,2,0,0,0,2,0,0,0,9,0,0,0,19, + 0,0,0,67,0,0,0,115,89,1,0,0,124,1,0,100, + 1,0,107,8,0,114,73,0,100,2,0,125,1,0,116,0, + 0,124,2,0,100,3,0,131,2,0,114,73,0,121,19,0, + 124,2,0,106,1,0,124,0,0,131,1,0,125,1,0,87, + 110,18,0,4,116,2,0,107,10,0,114,72,0,1,1,1, + 89,110,1,0,88,116,3,0,106,4,0,124,0,0,124,2, + 0,100,4,0,124,1,0,131,2,1,125,4,0,100,5,0, + 124,4,0,95,5,0,124,2,0,100,1,0,107,8,0,114, + 194,0,120,73,0,116,6,0,131,0,0,68,93,58,0,92, + 2,0,125,5,0,125,6,0,124,1,0,106,7,0,116,8, + 0,124,6,0,131,1,0,131,1,0,114,128,0,124,5,0, + 124,0,0,124,1,0,131,2,0,125,2,0,124,2,0,124, + 4,0,95,9,0,80,113,128,0,87,100,1,0,83,124,3, + 0,116,10,0,107,8,0,114,23,1,116,0,0,124,2,0, + 100,6,0,131,2,0,114,32,1,121,19,0,124,2,0,106, + 11,0,124,0,0,131,1,0,125,7,0,87,110,18,0,4, + 116,2,0,107,10,0,114,4,1,1,1,1,89,113,32,1, + 88,124,7,0,114,32,1,103,0,0,124,4,0,95,12,0, + 110,9,0,124,3,0,124,4,0,95,12,0,124,4,0,106, + 12,0,103,0,0,107,2,0,114,85,1,124,1,0,114,85, + 1,116,13,0,124,1,0,131,1,0,100,7,0,25,125,8, + 0,124,4,0,106,12,0,106,14,0,124,8,0,131,1,0, + 1,124,4,0,83,41,8,97,61,1,0,0,82,101,116,117, + 114,110,32,97,32,109,111,100,117,108,101,32,115,112,101,99, + 32,98,97,115,101,100,32,111,110,32,97,32,102,105,108,101, + 32,108,111,99,97,116,105,111,110,46,10,10,32,32,32,32, + 84,111,32,105,110,100,105,99,97,116,101,32,116,104,97,116, + 32,116,104,101,32,109,111,100,117,108,101,32,105,115,32,97, + 32,112,97,99,107,97,103,101,44,32,115,101,116,10,32,32, + 32,32,115,117,98,109,111,100,117,108,101,95,115,101,97,114, + 99,104,95,108,111,99,97,116,105,111,110,115,32,116,111,32, + 97,32,108,105,115,116,32,111,102,32,100,105,114,101,99,116, + 111,114,121,32,112,97,116,104,115,46,32,32,65,110,10,32, + 32,32,32,101,109,112,116,121,32,108,105,115,116,32,105,115, + 32,115,117,102,102,105,99,105,101,110,116,44,32,116,104,111, + 117,103,104,32,105,116,115,32,110,111,116,32,111,116,104,101, + 114,119,105,115,101,32,117,115,101,102,117,108,32,116,111,32, + 116,104,101,10,32,32,32,32,105,109,112,111,114,116,32,115, + 121,115,116,101,109,46,10,10,32,32,32,32,84,104,101,32, + 108,111,97,100,101,114,32,109,117,115,116,32,116,97,107,101, + 32,97,32,115,112,101,99,32,97,115,32,105,116,115,32,111, + 110,108,121,32,95,95,105,110,105,116,95,95,40,41,32,97, + 114,103,46,10,10,32,32,32,32,78,122,9,60,117,110,107, + 110,111,119,110,62,218,12,103,101,116,95,102,105,108,101,110, + 97,109,101,218,6,111,114,105,103,105,110,84,218,10,105,115, + 95,112,97,99,107,97,103,101,114,59,0,0,0,41,15,114, + 108,0,0,0,114,151,0,0,0,114,99,0,0,0,114,114, + 0,0,0,218,10,77,111,100,117,108,101,83,112,101,99,90, + 13,95,115,101,116,95,102,105,108,101,97,116,116,114,218,27, + 95,103,101,116,95,115,117,112,112,111,114,116,101,100,95,102, + 105,108,101,95,108,111,97,100,101,114,115,114,92,0,0,0, + 114,93,0,0,0,114,120,0,0,0,218,9,95,80,79,80, + 85,76,65,84,69,114,153,0,0,0,114,150,0,0,0,114, + 38,0,0,0,218,6,97,112,112,101,110,100,41,9,114,98, + 0,0,0,90,8,108,111,99,97,116,105,111,110,114,120,0, + 0,0,114,150,0,0,0,218,4,115,112,101,99,218,12,108, + 111,97,100,101,114,95,99,108,97,115,115,218,8,115,117,102, + 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, + 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, + 108,101,95,108,111,99,97,116,105,111,110,246,1,0,0,115, + 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, + 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, + 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, + 5,2,6,1,12,2,9,1,15,1,6,1,16,1,16,2, + 114,161,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,5,0,0,0,64,0,0,0,115,121,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,90,3, + 0,100,2,0,90,4,0,100,3,0,90,5,0,100,4,0, + 90,6,0,101,7,0,100,5,0,100,6,0,132,0,0,131, + 1,0,90,8,0,101,7,0,100,7,0,100,8,0,132,0, + 0,131,1,0,90,9,0,101,7,0,100,9,0,100,9,0, + 100,10,0,100,11,0,132,2,0,131,1,0,90,10,0,101, + 7,0,100,9,0,100,12,0,100,13,0,132,1,0,131,1, + 0,90,11,0,100,9,0,83,41,14,218,21,87,105,110,100, + 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, + 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, + 100,101,114,32,102,111,114,32,109,111,100,117,108,101,115,32, + 100,101,99,108,97,114,101,100,32,105,110,32,116,104,101,32, + 87,105,110,100,111,119,115,32,114,101,103,105,115,116,114,121, + 46,122,59,83,111,102,116,119,97,114,101,92,80,121,116,104, + 111,110,92,80,121,116,104,111,110,67,111,114,101,92,123,115, + 121,115,95,118,101,114,115,105,111,110,125,92,77,111,100,117, + 108,101,115,92,123,102,117,108,108,110,97,109,101,125,122,65, + 83,111,102,116,119,97,114,101,92,80,121,116,104,111,110,92, + 80,121,116,104,111,110,67,111,114,101,92,123,115,121,115,95, + 118,101,114,115,105,111,110,125,92,77,111,100,117,108,101,115, + 92,123,102,117,108,108,110,97,109,101,125,92,68,101,98,117, + 103,70,99,2,0,0,0,0,0,0,0,2,0,0,0,11, + 0,0,0,67,0,0,0,115,67,0,0,0,121,23,0,116, + 0,0,106,1,0,116,0,0,106,2,0,124,1,0,131,2, + 0,83,87,110,37,0,4,116,3,0,107,10,0,114,62,0, + 1,1,1,116,0,0,106,1,0,116,0,0,106,4,0,124, + 1,0,131,2,0,83,89,110,1,0,88,100,0,0,83,41, + 1,78,41,5,218,7,95,119,105,110,114,101,103,90,7,79, + 112,101,110,75,101,121,90,17,72,75,69,89,95,67,85,82, + 82,69,78,84,95,85,83,69,82,114,40,0,0,0,90,18, + 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, + 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, + 111,112,101,110,95,114,101,103,105,115,116,114,121,68,2,0, + 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, - 105,110,100,101,114,46,102,105,110,100,95,115,112,101,99,99, - 3,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, - 67,0,0,0,115,45,0,0,0,124,0,0,106,0,0,124, - 1,0,124,2,0,131,2,0,125,3,0,124,3,0,100,1, - 0,107,9,0,114,37,0,124,3,0,106,1,0,83,100,1, - 0,83,100,1,0,83,41,2,122,108,70,105,110,100,32,109, - 111,100,117,108,101,32,110,97,109,101,100,32,105,110,32,116, - 104,101,32,114,101,103,105,115,116,114,121,46,10,10,32,32, - 32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111, - 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, - 32,32,85,115,101,32,101,120,101,99,95,109,111,100,117,108, - 101,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32, - 32,32,32,32,32,32,78,41,2,114,181,0,0,0,114,127, - 0,0,0,41,4,114,170,0,0,0,114,126,0,0,0,114, - 35,0,0,0,114,164,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,11,102,105,110,100,95,109, - 111,100,117,108,101,114,2,0,0,115,8,0,0,0,0,7, - 18,1,12,1,7,2,122,33,87,105,110,100,111,119,115,82, - 101,103,105,115,116,114,121,70,105,110,100,101,114,46,102,105, - 110,100,95,109,111,100,117,108,101,41,12,114,112,0,0,0, - 114,111,0,0,0,114,113,0,0,0,114,114,0,0,0,114, - 175,0,0,0,114,174,0,0,0,114,173,0,0,0,218,11, - 99,108,97,115,115,109,101,116,104,111,100,114,172,0,0,0, - 114,178,0,0,0,114,181,0,0,0,114,182,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,168,0,0,0,64,2,0,0,115,20,0,0, - 0,12,2,6,3,6,3,6,2,6,2,18,7,18,15,3, - 1,21,15,3,1,114,168,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, - 70,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, - 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, - 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,132,0,0,90,6,0,100,8,0,100,9,0, - 132,0,0,90,7,0,100,10,0,83,41,11,218,13,95,76, - 111,97,100,101,114,66,97,115,105,99,115,122,83,66,97,115, - 101,32,99,108,97,115,115,32,111,102,32,99,111,109,109,111, - 110,32,99,111,100,101,32,110,101,101,100,101,100,32,98,121, - 32,98,111,116,104,32,83,111,117,114,99,101,76,111,97,100, - 101,114,32,97,110,100,10,32,32,32,32,83,111,117,114,99, - 101,108,101,115,115,70,105,108,101,76,111,97,100,101,114,46, - 99,2,0,0,0,0,0,0,0,5,0,0,0,3,0,0, - 0,67,0,0,0,115,88,0,0,0,116,0,0,124,0,0, - 106,1,0,124,1,0,131,1,0,131,1,0,100,1,0,25, - 125,2,0,124,2,0,106,2,0,100,2,0,100,1,0,131, - 2,0,100,3,0,25,125,3,0,124,1,0,106,3,0,100, - 2,0,131,1,0,100,4,0,25,125,4,0,124,3,0,100, - 5,0,107,2,0,111,87,0,124,4,0,100,5,0,107,3, - 0,83,41,6,122,141,67,111,110,99,114,101,116,101,32,105, - 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, - 32,73,110,115,112,101,99,116,76,111,97,100,101,114,46,105, - 115,95,112,97,99,107,97,103,101,32,98,121,32,99,104,101, - 99,107,105,110,103,32,105,102,10,32,32,32,32,32,32,32, - 32,116,104,101,32,112,97,116,104,32,114,101,116,117,114,110, - 101,100,32,98,121,32,103,101,116,95,102,105,108,101,110,97, - 109,101,32,104,97,115,32,97,32,102,105,108,101,110,97,109, - 101,32,111,102,32,39,95,95,105,110,105,116,95,95,46,112, - 121,39,46,114,29,0,0,0,114,58,0,0,0,114,59,0, - 0,0,114,56,0,0,0,218,8,95,95,105,110,105,116,95, - 95,41,4,114,38,0,0,0,114,157,0,0,0,114,34,0, - 0,0,114,32,0,0,0,41,5,114,108,0,0,0,114,126, - 0,0,0,114,94,0,0,0,90,13,102,105,108,101,110,97, - 109,101,95,98,97,115,101,90,9,116,97,105,108,95,110,97, - 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,159,0,0,0,133,2,0,0,115,8,0,0,0,0, - 3,25,1,22,1,19,1,122,24,95,76,111,97,100,101,114, - 66,97,115,105,99,115,46,105,115,95,112,97,99,107,97,103, - 101,99,2,0,0,0,0,0,0,0,2,0,0,0,1,0, - 0,0,67,0,0,0,115,4,0,0,0,100,1,0,83,41, - 2,122,42,85,115,101,32,100,101,102,97,117,108,116,32,115, - 101,109,97,110,116,105,99,115,32,102,111,114,32,109,111,100, - 117,108,101,32,99,114,101,97,116,105,111,110,46,78,114,4, - 0,0,0,41,2,114,108,0,0,0,114,164,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,13, - 99,114,101,97,116,101,95,109,111,100,117,108,101,141,2,0, - 0,115,0,0,0,0,122,27,95,76,111,97,100,101,114,66, - 97,115,105,99,115,46,99,114,101,97,116,101,95,109,111,100, - 117,108,101,99,2,0,0,0,0,0,0,0,3,0,0,0, - 4,0,0,0,67,0,0,0,115,80,0,0,0,124,0,0, - 106,0,0,124,1,0,106,1,0,131,1,0,125,2,0,124, - 2,0,100,1,0,107,8,0,114,54,0,116,2,0,100,2, - 0,106,3,0,124,1,0,106,1,0,131,1,0,131,1,0, - 130,1,0,116,4,0,106,5,0,116,6,0,124,2,0,124, - 1,0,106,7,0,131,3,0,1,100,1,0,83,41,3,122, - 19,69,120,101,99,117,116,101,32,116,104,101,32,109,111,100, - 117,108,101,46,78,122,52,99,97,110,110,111,116,32,108,111, - 97,100,32,109,111,100,117,108,101,32,123,33,114,125,32,119, - 104,101,110,32,103,101,116,95,99,111,100,101,40,41,32,114, - 101,116,117,114,110,115,32,78,111,110,101,41,8,218,8,103, - 101,116,95,99,111,100,101,114,112,0,0,0,114,107,0,0, - 0,114,47,0,0,0,114,121,0,0,0,218,25,95,99,97, - 108,108,95,119,105,116,104,95,102,114,97,109,101,115,95,114, - 101,109,111,118,101,100,218,4,101,120,101,99,114,118,0,0, - 0,41,3,114,108,0,0,0,218,6,109,111,100,117,108,101, - 114,146,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,11,101,120,101,99,95,109,111,100,117,108, - 101,144,2,0,0,115,10,0,0,0,0,2,18,1,12,1, - 9,1,15,1,122,25,95,76,111,97,100,101,114,66,97,115, - 105,99,115,46,101,120,101,99,95,109,111,100,117,108,101,99, - 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, - 67,0,0,0,115,16,0,0,0,116,0,0,106,1,0,124, - 0,0,124,1,0,131,2,0,83,41,1,78,41,2,114,121, - 0,0,0,218,17,95,108,111,97,100,95,109,111,100,117,108, - 101,95,115,104,105,109,41,2,114,108,0,0,0,114,126,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,108,111,97,100,95,109,111,100,117,108,101,152,2, - 0,0,115,2,0,0,0,0,1,122,25,95,76,111,97,100, - 101,114,66,97,115,105,99,115,46,108,111,97,100,95,109,111, - 100,117,108,101,78,41,8,114,112,0,0,0,114,111,0,0, - 0,114,113,0,0,0,114,114,0,0,0,114,159,0,0,0, - 114,186,0,0,0,114,191,0,0,0,114,193,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,184,0,0,0,128,2,0,0,115,10,0,0, - 0,12,3,6,2,12,8,12,3,12,8,114,184,0,0,0, - 99,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0, - 0,64,0,0,0,115,106,0,0,0,101,0,0,90,1,0, - 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, - 3,0,100,3,0,100,4,0,132,0,0,90,4,0,100,5, - 0,100,6,0,132,0,0,90,5,0,100,7,0,100,8,0, - 132,0,0,90,6,0,100,9,0,100,10,0,132,0,0,90, - 7,0,100,11,0,100,18,0,100,13,0,100,14,0,132,0, - 1,90,8,0,100,15,0,100,16,0,132,0,0,90,9,0, - 100,17,0,83,41,19,218,12,83,111,117,114,99,101,76,111, - 97,100,101,114,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,10,0,0,0,116,0, - 0,130,1,0,100,1,0,83,41,2,122,178,79,112,116,105, - 111,110,97,108,32,109,101,116,104,111,100,32,116,104,97,116, - 32,114,101,116,117,114,110,115,32,116,104,101,32,109,111,100, - 105,102,105,99,97,116,105,111,110,32,116,105,109,101,32,40, - 97,110,32,105,110,116,41,32,102,111,114,32,116,104,101,10, - 32,32,32,32,32,32,32,32,115,112,101,99,105,102,105,101, - 100,32,112,97,116,104,44,32,119,104,101,114,101,32,112,97, - 116,104,32,105,115,32,97,32,115,116,114,46,10,10,32,32, - 32,32,32,32,32,32,82,97,105,115,101,115,32,73,79,69, - 114,114,111,114,32,119,104,101,110,32,116,104,101,32,112,97, - 116,104,32,99,97,110,110,111,116,32,98,101,32,104,97,110, - 100,108,101,100,46,10,32,32,32,32,32,32,32,32,78,41, - 1,218,7,73,79,69,114,114,111,114,41,2,114,108,0,0, - 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,10,112,97,116,104,95,109,116,105,109, - 101,158,2,0,0,115,2,0,0,0,0,6,122,23,83,111, - 117,114,99,101,76,111,97,100,101,114,46,112,97,116,104,95, - 109,116,105,109,101,99,2,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,19,0,0,0,100, - 1,0,124,0,0,106,0,0,124,1,0,131,1,0,105,1, - 0,83,41,2,97,170,1,0,0,79,112,116,105,111,110,97, - 108,32,109,101,116,104,111,100,32,114,101,116,117,114,110,105, - 110,103,32,97,32,109,101,116,97,100,97,116,97,32,100,105, - 99,116,32,102,111,114,32,116,104,101,32,115,112,101,99,105, - 102,105,101,100,32,112,97,116,104,10,32,32,32,32,32,32, - 32,32,116,111,32,98,121,32,116,104,101,32,112,97,116,104, - 32,40,115,116,114,41,46,10,32,32,32,32,32,32,32,32, - 80,111,115,115,105,98,108,101,32,107,101,121,115,58,10,32, - 32,32,32,32,32,32,32,45,32,39,109,116,105,109,101,39, - 32,40,109,97,110,100,97,116,111,114,121,41,32,105,115,32, - 116,104,101,32,110,117,109,101,114,105,99,32,116,105,109,101, - 115,116,97,109,112,32,111,102,32,108,97,115,116,32,115,111, - 117,114,99,101,10,32,32,32,32,32,32,32,32,32,32,99, - 111,100,101,32,109,111,100,105,102,105,99,97,116,105,111,110, - 59,10,32,32,32,32,32,32,32,32,45,32,39,115,105,122, - 101,39,32,40,111,112,116,105,111,110,97,108,41,32,105,115, - 32,116,104,101,32,115,105,122,101,32,105,110,32,98,121,116, - 101,115,32,111,102,32,116,104,101,32,115,111,117,114,99,101, - 32,99,111,100,101,46,10,10,32,32,32,32,32,32,32,32, - 73,109,112,108,101,109,101,110,116,105,110,103,32,116,104,105, - 115,32,109,101,116,104,111,100,32,97,108,108,111,119,115,32, - 116,104,101,32,108,111,97,100,101,114,32,116,111,32,114,101, - 97,100,32,98,121,116,101,99,111,100,101,32,102,105,108,101, - 115,46,10,32,32,32,32,32,32,32,32,82,97,105,115,101, - 115,32,73,79,69,114,114,111,114,32,119,104,101,110,32,116, - 104,101,32,112,97,116,104,32,99,97,110,110,111,116,32,98, - 101,32,104,97,110,100,108,101,100,46,10,32,32,32,32,32, - 32,32,32,114,133,0,0,0,41,1,114,196,0,0,0,41, - 2,114,108,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,10,112,97,116,104, - 95,115,116,97,116,115,166,2,0,0,115,2,0,0,0,0, - 11,122,23,83,111,117,114,99,101,76,111,97,100,101,114,46, - 112,97,116,104,95,115,116,97,116,115,99,4,0,0,0,0, - 0,0,0,4,0,0,0,3,0,0,0,67,0,0,0,115, - 16,0,0,0,124,0,0,106,0,0,124,2,0,124,3,0, - 131,2,0,83,41,1,122,228,79,112,116,105,111,110,97,108, - 32,109,101,116,104,111,100,32,119,104,105,99,104,32,119,114, - 105,116,101,115,32,100,97,116,97,32,40,98,121,116,101,115, - 41,32,116,111,32,97,32,102,105,108,101,32,112,97,116,104, - 32,40,97,32,115,116,114,41,46,10,10,32,32,32,32,32, - 32,32,32,73,109,112,108,101,109,101,110,116,105,110,103,32, - 116,104,105,115,32,109,101,116,104,111,100,32,97,108,108,111, - 119,115,32,102,111,114,32,116,104,101,32,119,114,105,116,105, - 110,103,32,111,102,32,98,121,116,101,99,111,100,101,32,102, - 105,108,101,115,46,10,10,32,32,32,32,32,32,32,32,84, - 104,101,32,115,111,117,114,99,101,32,112,97,116,104,32,105, - 115,32,110,101,101,100,101,100,32,105,110,32,111,114,100,101, - 114,32,116,111,32,99,111,114,114,101,99,116,108,121,32,116, - 114,97,110,115,102,101,114,32,112,101,114,109,105,115,115,105, - 111,110,115,10,32,32,32,32,32,32,32,32,41,1,218,8, - 115,101,116,95,100,97,116,97,41,4,114,108,0,0,0,114, - 90,0,0,0,90,10,99,97,99,104,101,95,112,97,116,104, - 114,53,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,15,95,99,97,99,104,101,95,98,121,116, - 101,99,111,100,101,179,2,0,0,115,2,0,0,0,0,8, - 122,28,83,111,117,114,99,101,76,111,97,100,101,114,46,95, - 99,97,99,104,101,95,98,121,116,101,99,111,100,101,99,3, - 0,0,0,0,0,0,0,3,0,0,0,1,0,0,0,67, - 0,0,0,115,4,0,0,0,100,1,0,83,41,2,122,150, - 79,112,116,105,111,110,97,108,32,109,101,116,104,111,100,32, - 119,104,105,99,104,32,119,114,105,116,101,115,32,100,97,116, - 97,32,40,98,121,116,101,115,41,32,116,111,32,97,32,102, - 105,108,101,32,112,97,116,104,32,40,97,32,115,116,114,41, - 46,10,10,32,32,32,32,32,32,32,32,73,109,112,108,101, - 109,101,110,116,105,110,103,32,116,104,105,115,32,109,101,116, - 104,111,100,32,97,108,108,111,119,115,32,102,111,114,32,116, - 104,101,32,119,114,105,116,105,110,103,32,111,102,32,98,121, - 116,101,99,111,100,101,32,102,105,108,101,115,46,10,32,32, - 32,32,32,32,32,32,78,114,4,0,0,0,41,3,114,108, - 0,0,0,114,35,0,0,0,114,53,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,198,0,0, - 0,189,2,0,0,115,0,0,0,0,122,21,83,111,117,114, - 99,101,76,111,97,100,101,114,46,115,101,116,95,100,97,116, - 97,99,2,0,0,0,0,0,0,0,5,0,0,0,16,0, - 0,0,67,0,0,0,115,105,0,0,0,124,0,0,106,0, - 0,124,1,0,131,1,0,125,2,0,121,19,0,124,0,0, - 106,1,0,124,2,0,131,1,0,125,3,0,87,110,58,0, - 4,116,2,0,107,10,0,114,94,0,1,125,4,0,1,122, - 26,0,116,3,0,100,1,0,100,2,0,124,1,0,131,1, - 1,124,4,0,130,2,0,87,89,100,3,0,100,3,0,125, - 4,0,126,4,0,88,110,1,0,88,116,4,0,124,3,0, - 131,1,0,83,41,4,122,52,67,111,110,99,114,101,116,101, - 32,105,109,112,108,101,109,101,110,116,97,116,105,111,110,32, - 111,102,32,73,110,115,112,101,99,116,76,111,97,100,101,114, - 46,103,101,116,95,115,111,117,114,99,101,46,122,39,115,111, - 117,114,99,101,32,110,111,116,32,97,118,97,105,108,97,98, - 108,101,32,116,104,114,111,117,103,104,32,103,101,116,95,100, - 97,116,97,40,41,114,106,0,0,0,78,41,5,114,157,0, - 0,0,218,8,103,101,116,95,100,97,116,97,114,40,0,0, - 0,114,107,0,0,0,114,155,0,0,0,41,5,114,108,0, - 0,0,114,126,0,0,0,114,35,0,0,0,114,153,0,0, - 0,218,3,101,120,99,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,10,103,101,116,95,115,111,117,114,99, - 101,196,2,0,0,115,14,0,0,0,0,2,15,1,3,1, - 19,1,18,1,9,1,31,1,122,23,83,111,117,114,99,101, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,218,9,95,111,112,116,105,109,105,122,101,114,29,0,0, - 0,99,3,0,0,0,1,0,0,0,4,0,0,0,9,0, - 0,0,67,0,0,0,115,34,0,0,0,116,0,0,106,1, - 0,116,2,0,124,1,0,124,2,0,100,1,0,100,2,0, - 100,3,0,100,4,0,124,3,0,131,4,2,83,41,5,122, - 130,82,101,116,117,114,110,32,116,104,101,32,99,111,100,101, - 32,111,98,106,101,99,116,32,99,111,109,112,105,108,101,100, - 32,102,114,111,109,32,115,111,117,114,99,101,46,10,10,32, - 32,32,32,32,32,32,32,84,104,101,32,39,100,97,116,97, - 39,32,97,114,103,117,109,101,110,116,32,99,97,110,32,98, - 101,32,97,110,121,32,111,98,106,101,99,116,32,116,121,112, - 101,32,116,104,97,116,32,99,111,109,112,105,108,101,40,41, - 32,115,117,112,112,111,114,116,115,46,10,32,32,32,32,32, - 32,32,32,114,189,0,0,0,218,12,100,111,110,116,95,105, - 110,104,101,114,105,116,84,114,68,0,0,0,41,3,114,121, - 0,0,0,114,188,0,0,0,218,7,99,111,109,112,105,108, - 101,41,4,114,108,0,0,0,114,53,0,0,0,114,35,0, - 0,0,114,203,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,14,115,111,117,114,99,101,95,116, - 111,95,99,111,100,101,206,2,0,0,115,4,0,0,0,0, - 5,21,1,122,27,83,111,117,114,99,101,76,111,97,100,101, - 114,46,115,111,117,114,99,101,95,116,111,95,99,111,100,101, - 99,2,0,0,0,0,0,0,0,10,0,0,0,43,0,0, - 0,67,0,0,0,115,174,1,0,0,124,0,0,106,0,0, - 124,1,0,131,1,0,125,2,0,100,1,0,125,3,0,121, - 16,0,116,1,0,124,2,0,131,1,0,125,4,0,87,110, - 24,0,4,116,2,0,107,10,0,114,63,0,1,1,1,100, - 1,0,125,4,0,89,110,202,0,88,121,19,0,124,0,0, - 106,3,0,124,2,0,131,1,0,125,5,0,87,110,18,0, - 4,116,4,0,107,10,0,114,103,0,1,1,1,89,110,162, - 0,88,116,5,0,124,5,0,100,2,0,25,131,1,0,125, - 3,0,121,19,0,124,0,0,106,6,0,124,4,0,131,1, - 0,125,6,0,87,110,18,0,4,116,7,0,107,10,0,114, - 159,0,1,1,1,89,110,106,0,88,121,34,0,116,8,0, - 124,6,0,100,3,0,124,5,0,100,4,0,124,1,0,100, - 5,0,124,4,0,131,1,3,125,7,0,87,110,24,0,4, - 116,9,0,116,10,0,102,2,0,107,10,0,114,220,0,1, - 1,1,89,110,45,0,88,116,11,0,100,6,0,124,4,0, - 124,2,0,131,3,0,1,116,12,0,124,7,0,100,4,0, - 124,1,0,100,7,0,124,4,0,100,8,0,124,2,0,131, - 1,3,83,124,0,0,106,6,0,124,2,0,131,1,0,125, - 8,0,124,0,0,106,13,0,124,8,0,124,2,0,131,2, - 0,125,9,0,116,11,0,100,9,0,124,2,0,131,2,0, - 1,116,14,0,106,15,0,12,114,170,1,124,4,0,100,1, - 0,107,9,0,114,170,1,124,3,0,100,1,0,107,9,0, - 114,170,1,116,16,0,124,9,0,124,3,0,116,17,0,124, - 8,0,131,1,0,131,3,0,125,6,0,121,36,0,124,0, - 0,106,18,0,124,2,0,124,4,0,124,6,0,131,3,0, - 1,116,11,0,100,10,0,124,4,0,131,2,0,1,87,110, - 18,0,4,116,2,0,107,10,0,114,169,1,1,1,1,89, - 110,1,0,88,124,9,0,83,41,11,122,190,67,111,110,99, + 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, + 115,116,114,121,99,2,0,0,0,0,0,0,0,6,0,0, + 0,16,0,0,0,67,0,0,0,115,143,0,0,0,124,0, + 0,106,0,0,114,21,0,124,0,0,106,1,0,125,2,0, + 110,9,0,124,0,0,106,2,0,125,2,0,124,2,0,106, + 3,0,100,1,0,124,1,0,100,2,0,116,4,0,106,5, + 0,100,0,0,100,3,0,133,2,0,25,131,0,2,125,3, + 0,121,47,0,124,0,0,106,6,0,124,3,0,131,1,0, + 143,25,0,125,4,0,116,7,0,106,8,0,124,4,0,100, + 4,0,131,2,0,125,5,0,87,100,0,0,81,82,88,87, + 110,22,0,4,116,9,0,107,10,0,114,138,0,1,1,1, + 100,0,0,83,89,110,1,0,88,124,5,0,83,41,5,78, + 114,119,0,0,0,90,11,115,121,115,95,118,101,114,115,105, + 111,110,114,80,0,0,0,114,30,0,0,0,41,10,218,11, + 68,69,66,85,71,95,66,85,73,76,68,218,18,82,69,71, + 73,83,84,82,89,95,75,69,89,95,68,69,66,85,71,218, + 12,82,69,71,73,83,84,82,89,95,75,69,89,114,47,0, + 0,0,114,7,0,0,0,218,7,118,101,114,115,105,111,110, + 114,166,0,0,0,114,163,0,0,0,90,10,81,117,101,114, + 121,86,97,108,117,101,114,40,0,0,0,41,6,114,164,0, + 0,0,114,119,0,0,0,90,12,114,101,103,105,115,116,114, + 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, + 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, + 99,104,95,114,101,103,105,115,116,114,121,75,2,0,0,115, + 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, + 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, + 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, + 114,46,95,115,101,97,114,99,104,95,114,101,103,105,115,116, + 114,121,78,99,4,0,0,0,0,0,0,0,8,0,0,0, + 14,0,0,0,67,0,0,0,115,158,0,0,0,124,0,0, + 106,0,0,124,1,0,131,1,0,125,4,0,124,4,0,100, + 0,0,107,8,0,114,31,0,100,0,0,83,121,14,0,116, + 1,0,124,4,0,131,1,0,1,87,110,22,0,4,116,2, + 0,107,10,0,114,69,0,1,1,1,100,0,0,83,89,110, + 1,0,88,120,81,0,116,3,0,131,0,0,68,93,70,0, + 92,2,0,125,5,0,125,6,0,124,4,0,106,4,0,116, + 5,0,124,6,0,131,1,0,131,1,0,114,80,0,116,6, + 0,106,7,0,124,1,0,124,5,0,124,1,0,124,4,0, + 131,2,0,100,1,0,124,4,0,131,2,1,125,7,0,124, + 7,0,83,113,80,0,87,100,0,0,83,41,2,78,114,152, + 0,0,0,41,8,114,172,0,0,0,114,39,0,0,0,114, + 40,0,0,0,114,155,0,0,0,114,92,0,0,0,114,93, + 0,0,0,114,114,0,0,0,218,16,115,112,101,99,95,102, + 114,111,109,95,108,111,97,100,101,114,41,8,114,164,0,0, + 0,114,119,0,0,0,114,35,0,0,0,218,6,116,97,114, + 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, + 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, + 99,90,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, + 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, + 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, + 95,115,112,101,99,99,3,0,0,0,0,0,0,0,4,0, + 0,0,3,0,0,0,67,0,0,0,115,45,0,0,0,124, + 0,0,106,0,0,124,1,0,124,2,0,131,2,0,125,3, + 0,124,3,0,100,1,0,107,9,0,114,37,0,124,3,0, + 106,1,0,83,100,1,0,83,100,1,0,83,41,2,122,108, + 70,105,110,100,32,109,111,100,117,108,101,32,110,97,109,101, + 100,32,105,110,32,116,104,101,32,114,101,103,105,115,116,114, + 121,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, + 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, + 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, + 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, + 100,46,10,10,32,32,32,32,32,32,32,32,78,41,2,114, + 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, + 102,105,110,100,95,109,111,100,117,108,101,106,2,0,0,115, + 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, + 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, + 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, + 12,114,105,0,0,0,114,104,0,0,0,114,106,0,0,0, + 114,107,0,0,0,114,169,0,0,0,114,168,0,0,0,114, + 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, + 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, + 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,56,2, + 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, + 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, + 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, + 0,64,0,0,0,115,70,0,0,0,101,0,0,90,1,0, + 100,0,0,90,2,0,100,1,0,90,3,0,100,2,0,100, + 3,0,132,0,0,90,4,0,100,4,0,100,5,0,132,0, + 0,90,5,0,100,6,0,100,7,0,132,0,0,90,6,0, + 100,8,0,100,9,0,132,0,0,90,7,0,100,10,0,83, + 41,11,218,13,95,76,111,97,100,101,114,66,97,115,105,99, + 115,122,83,66,97,115,101,32,99,108,97,115,115,32,111,102, + 32,99,111,109,109,111,110,32,99,111,100,101,32,110,101,101, + 100,101,100,32,98,121,32,98,111,116,104,32,83,111,117,114, + 99,101,76,111,97,100,101,114,32,97,110,100,10,32,32,32, + 32,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, + 111,97,100,101,114,46,99,2,0,0,0,0,0,0,0,5, + 0,0,0,3,0,0,0,67,0,0,0,115,88,0,0,0, + 116,0,0,124,0,0,106,1,0,124,1,0,131,1,0,131, + 1,0,100,1,0,25,125,2,0,124,2,0,106,2,0,100, + 2,0,100,1,0,131,2,0,100,3,0,25,125,3,0,124, + 1,0,106,3,0,100,2,0,131,1,0,100,4,0,25,125, + 4,0,124,3,0,100,5,0,107,2,0,111,87,0,124,4, + 0,100,5,0,107,3,0,83,41,6,122,141,67,111,110,99, 114,101,116,101,32,105,109,112,108,101,109,101,110,116,97,116, 105,111,110,32,111,102,32,73,110,115,112,101,99,116,76,111, - 97,100,101,114,46,103,101,116,95,99,111,100,101,46,10,10, - 32,32,32,32,32,32,32,32,82,101,97,100,105,110,103,32, - 111,102,32,98,121,116,101,99,111,100,101,32,114,101,113,117, - 105,114,101,115,32,112,97,116,104,95,115,116,97,116,115,32, - 116,111,32,98,101,32,105,109,112,108,101,109,101,110,116,101, - 100,46,32,84,111,32,119,114,105,116,101,10,32,32,32,32, - 32,32,32,32,98,121,116,101,99,111,100,101,44,32,115,101, - 116,95,100,97,116,97,32,109,117,115,116,32,97,108,115,111, - 32,98,101,32,105,109,112,108,101,109,101,110,116,101,100,46, - 10,10,32,32,32,32,32,32,32,32,78,114,133,0,0,0, - 114,138,0,0,0,114,106,0,0,0,114,35,0,0,0,122, - 13,123,125,32,109,97,116,99,104,101,115,32,123,125,114,89, - 0,0,0,114,90,0,0,0,122,19,99,111,100,101,32,111, - 98,106,101,99,116,32,102,114,111,109,32,123,125,122,10,119, - 114,111,116,101,32,123,33,114,125,41,19,114,157,0,0,0, - 114,79,0,0,0,114,66,0,0,0,114,197,0,0,0,114, - 195,0,0,0,114,14,0,0,0,114,200,0,0,0,114,40, - 0,0,0,114,141,0,0,0,114,107,0,0,0,114,136,0, - 0,0,114,105,0,0,0,114,147,0,0,0,114,206,0,0, - 0,114,7,0,0,0,218,19,100,111,110,116,95,119,114,105, - 116,101,95,98,121,116,101,99,111,100,101,114,150,0,0,0, - 114,31,0,0,0,114,199,0,0,0,41,10,114,108,0,0, - 0,114,126,0,0,0,114,90,0,0,0,114,139,0,0,0, - 114,89,0,0,0,218,2,115,116,114,53,0,0,0,218,10, - 98,121,116,101,115,95,100,97,116,97,114,153,0,0,0,90, - 11,99,111,100,101,95,111,98,106,101,99,116,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,187,0,0,0, - 214,2,0,0,115,78,0,0,0,0,7,15,1,6,1,3, - 1,16,1,13,1,11,2,3,1,19,1,13,1,5,2,16, - 1,3,1,19,1,13,1,5,2,3,1,9,1,12,1,13, - 1,19,1,5,2,9,1,7,1,15,1,6,1,7,1,15, - 1,18,1,13,1,22,1,12,1,9,1,15,1,3,1,19, - 1,17,1,13,1,5,1,122,21,83,111,117,114,99,101,76, - 111,97,100,101,114,46,103,101,116,95,99,111,100,101,78,114, - 87,0,0,0,41,10,114,112,0,0,0,114,111,0,0,0, - 114,113,0,0,0,114,196,0,0,0,114,197,0,0,0,114, - 199,0,0,0,114,198,0,0,0,114,202,0,0,0,114,206, - 0,0,0,114,187,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,194,0,0, - 0,156,2,0,0,115,14,0,0,0,12,2,12,8,12,13, - 12,10,12,7,12,10,18,8,114,194,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0, - 0,0,115,112,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,100,2,0,100,3,0,132, - 0,0,90,4,0,100,4,0,100,5,0,132,0,0,90,5, - 0,100,6,0,100,7,0,132,0,0,90,6,0,101,7,0, - 135,0,0,102,1,0,100,8,0,100,9,0,134,0,0,131, - 1,0,90,8,0,101,7,0,100,10,0,100,11,0,132,0, - 0,131,1,0,90,9,0,100,12,0,100,13,0,132,0,0, - 90,10,0,135,0,0,83,41,14,218,10,70,105,108,101,76, - 111,97,100,101,114,122,103,66,97,115,101,32,102,105,108,101, - 32,108,111,97,100,101,114,32,99,108,97,115,115,32,119,104, - 105,99,104,32,105,109,112,108,101,109,101,110,116,115,32,116, - 104,101,32,108,111,97,100,101,114,32,112,114,111,116,111,99, - 111,108,32,109,101,116,104,111,100,115,32,116,104,97,116,10, - 32,32,32,32,114,101,113,117,105,114,101,32,102,105,108,101, - 32,115,121,115,116,101,109,32,117,115,97,103,101,46,99,3, - 0,0,0,0,0,0,0,3,0,0,0,2,0,0,0,67, - 0,0,0,115,22,0,0,0,124,1,0,124,0,0,95,0, - 0,124,2,0,124,0,0,95,1,0,100,1,0,83,41,2, - 122,75,67,97,99,104,101,32,116,104,101,32,109,111,100,117, - 108,101,32,110,97,109,101,32,97,110,100,32,116,104,101,32, - 112,97,116,104,32,116,111,32,116,104,101,32,102,105,108,101, - 32,102,111,117,110,100,32,98,121,32,116,104,101,10,32,32, - 32,32,32,32,32,32,102,105,110,100,101,114,46,78,41,2, - 114,106,0,0,0,114,35,0,0,0,41,3,114,108,0,0, - 0,114,126,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,185,0,0,0,15, - 3,0,0,115,4,0,0,0,0,3,9,1,122,19,70,105, - 108,101,76,111,97,100,101,114,46,95,95,105,110,105,116,95, - 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, - 0,0,67,0,0,0,115,34,0,0,0,124,0,0,106,0, - 0,124,1,0,106,0,0,107,2,0,111,33,0,124,0,0, - 106,1,0,124,1,0,106,1,0,107,2,0,83,41,1,78, - 41,2,218,9,95,95,99,108,97,115,115,95,95,114,118,0, - 0,0,41,2,114,108,0,0,0,218,5,111,116,104,101,114, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, - 6,95,95,101,113,95,95,21,3,0,0,115,4,0,0,0, - 0,1,18,1,122,17,70,105,108,101,76,111,97,100,101,114, - 46,95,95,101,113,95,95,99,1,0,0,0,0,0,0,0, - 1,0,0,0,3,0,0,0,67,0,0,0,115,26,0,0, - 0,116,0,0,124,0,0,106,1,0,131,1,0,116,0,0, - 124,0,0,106,2,0,131,1,0,65,83,41,1,78,41,3, - 218,4,104,97,115,104,114,106,0,0,0,114,35,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,8,95,95,104,97,115,104,95,95, - 25,3,0,0,115,2,0,0,0,0,1,122,19,70,105,108, - 101,76,111,97,100,101,114,46,95,95,104,97,115,104,95,95, - 99,2,0,0,0,0,0,0,0,2,0,0,0,3,0,0, - 0,3,0,0,0,115,22,0,0,0,116,0,0,116,1,0, - 124,0,0,131,2,0,106,2,0,124,1,0,131,1,0,83, - 41,1,122,100,76,111,97,100,32,97,32,109,111,100,117,108, - 101,32,102,114,111,109,32,97,32,102,105,108,101,46,10,10, - 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, - 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, - 32,32,32,32,32,32,32,32,41,3,218,5,115,117,112,101, - 114,114,210,0,0,0,114,193,0,0,0,41,2,114,108,0, - 0,0,114,126,0,0,0,41,1,114,211,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,28,3,0, - 0,115,2,0,0,0,0,10,122,22,70,105,108,101,76,111, - 97,100,101,114,46,108,111,97,100,95,109,111,100,117,108,101, - 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, - 0,67,0,0,0,115,7,0,0,0,124,0,0,106,0,0, - 83,41,1,122,58,82,101,116,117,114,110,32,116,104,101,32, - 112,97,116,104,32,116,111,32,116,104,101,32,115,111,117,114, - 99,101,32,102,105,108,101,32,97,115,32,102,111,117,110,100, - 32,98,121,32,116,104,101,32,102,105,110,100,101,114,46,41, - 1,114,35,0,0,0,41,2,114,108,0,0,0,114,126,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,157,0,0,0,40,3,0,0,115,2,0,0,0,0, - 3,122,23,70,105,108,101,76,111,97,100,101,114,46,103,101, - 116,95,102,105,108,101,110,97,109,101,99,2,0,0,0,0, - 0,0,0,3,0,0,0,9,0,0,0,67,0,0,0,115, - 42,0,0,0,116,0,0,106,1,0,124,1,0,100,1,0, - 131,2,0,143,17,0,125,2,0,124,2,0,106,2,0,131, - 0,0,83,87,100,2,0,81,82,88,100,2,0,83,41,3, - 122,39,82,101,116,117,114,110,32,116,104,101,32,100,97,116, - 97,32,102,114,111,109,32,112,97,116,104,32,97,115,32,114, - 97,119,32,98,121,116,101,115,46,218,1,114,78,41,3,114, - 49,0,0,0,114,50,0,0,0,90,4,114,101,97,100,41, - 3,114,108,0,0,0,114,35,0,0,0,114,54,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 200,0,0,0,45,3,0,0,115,4,0,0,0,0,2,21, - 1,122,19,70,105,108,101,76,111,97,100,101,114,46,103,101, - 116,95,100,97,116,97,41,11,114,112,0,0,0,114,111,0, - 0,0,114,113,0,0,0,114,114,0,0,0,114,185,0,0, - 0,114,213,0,0,0,114,215,0,0,0,114,123,0,0,0, - 114,193,0,0,0,114,157,0,0,0,114,200,0,0,0,114, - 4,0,0,0,114,4,0,0,0,41,1,114,211,0,0,0, - 114,5,0,0,0,114,210,0,0,0,10,3,0,0,115,14, - 0,0,0,12,3,6,2,12,6,12,4,12,3,24,12,18, - 5,114,210,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,4,0,0,0,64,0,0,0,115,64,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, - 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, - 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, - 100,8,0,100,9,0,132,0,1,90,6,0,100,10,0,83, - 41,11,218,16,83,111,117,114,99,101,70,105,108,101,76,111, - 97,100,101,114,122,62,67,111,110,99,114,101,116,101,32,105, - 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, - 32,83,111,117,114,99,101,76,111,97,100,101,114,32,117,115, - 105,110,103,32,116,104,101,32,102,105,108,101,32,115,121,115, - 116,101,109,46,99,2,0,0,0,0,0,0,0,3,0,0, - 0,4,0,0,0,67,0,0,0,115,34,0,0,0,116,0, - 0,124,1,0,131,1,0,125,2,0,100,1,0,124,2,0, - 106,1,0,100,2,0,124,2,0,106,2,0,105,2,0,83, - 41,3,122,33,82,101,116,117,114,110,32,116,104,101,32,109, - 101,116,97,100,97,116,97,32,102,111,114,32,116,104,101,32, - 112,97,116,104,46,114,133,0,0,0,114,134,0,0,0,41, - 3,114,39,0,0,0,218,8,115,116,95,109,116,105,109,101, - 90,7,115,116,95,115,105,122,101,41,3,114,108,0,0,0, - 114,35,0,0,0,114,208,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,197,0,0,0,55,3, - 0,0,115,4,0,0,0,0,2,12,1,122,27,83,111,117, - 114,99,101,70,105,108,101,76,111,97,100,101,114,46,112,97, - 116,104,95,115,116,97,116,115,99,4,0,0,0,0,0,0, - 0,5,0,0,0,5,0,0,0,67,0,0,0,115,34,0, - 0,0,116,0,0,124,1,0,131,1,0,125,4,0,124,0, - 0,106,1,0,124,2,0,124,3,0,100,1,0,124,4,0, - 131,2,1,83,41,2,78,218,5,95,109,111,100,101,41,2, - 114,97,0,0,0,114,198,0,0,0,41,5,114,108,0,0, - 0,114,90,0,0,0,114,89,0,0,0,114,53,0,0,0, - 114,42,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,199,0,0,0,60,3,0,0,115,4,0, - 0,0,0,2,12,1,122,32,83,111,117,114,99,101,70,105, - 108,101,76,111,97,100,101,114,46,95,99,97,99,104,101,95, - 98,121,116,101,99,111,100,101,114,220,0,0,0,105,182,1, - 0,0,99,3,0,0,0,1,0,0,0,9,0,0,0,17, - 0,0,0,67,0,0,0,115,53,1,0,0,116,0,0,124, - 1,0,131,1,0,92,2,0,125,4,0,125,5,0,103,0, - 0,125,6,0,120,54,0,124,4,0,114,80,0,116,1,0, - 124,4,0,131,1,0,12,114,80,0,116,0,0,124,4,0, - 131,1,0,92,2,0,125,4,0,125,7,0,124,6,0,106, - 2,0,124,7,0,131,1,0,1,113,27,0,87,120,132,0, - 116,3,0,124,6,0,131,1,0,68,93,118,0,125,7,0, - 116,4,0,124,4,0,124,7,0,131,2,0,125,4,0,121, - 17,0,116,5,0,106,6,0,124,4,0,131,1,0,1,87, - 113,94,0,4,116,7,0,107,10,0,114,155,0,1,1,1, - 119,94,0,89,113,94,0,4,116,8,0,107,10,0,114,211, - 0,1,125,8,0,1,122,25,0,116,9,0,100,1,0,124, - 4,0,124,8,0,131,3,0,1,100,2,0,83,87,89,100, - 2,0,100,2,0,125,8,0,126,8,0,88,113,94,0,88, - 113,94,0,87,121,33,0,116,10,0,124,1,0,124,2,0, - 124,3,0,131,3,0,1,116,9,0,100,3,0,124,1,0, - 131,2,0,1,87,110,53,0,4,116,8,0,107,10,0,114, - 48,1,1,125,8,0,1,122,21,0,116,9,0,100,1,0, - 124,1,0,124,8,0,131,3,0,1,87,89,100,2,0,100, - 2,0,125,8,0,126,8,0,88,110,1,0,88,100,2,0, - 83,41,4,122,27,87,114,105,116,101,32,98,121,116,101,115, - 32,100,97,116,97,32,116,111,32,97,32,102,105,108,101,46, - 122,27,99,111,117,108,100,32,110,111,116,32,99,114,101,97, - 116,101,32,123,33,114,125,58,32,123,33,114,125,78,122,12, - 99,114,101,97,116,101,100,32,123,33,114,125,41,11,114,38, - 0,0,0,114,46,0,0,0,114,163,0,0,0,114,33,0, - 0,0,114,28,0,0,0,114,3,0,0,0,90,5,109,107, - 100,105,114,218,15,70,105,108,101,69,120,105,115,116,115,69, - 114,114,111,114,114,40,0,0,0,114,105,0,0,0,114,55, - 0,0,0,41,9,114,108,0,0,0,114,35,0,0,0,114, - 53,0,0,0,114,220,0,0,0,218,6,112,97,114,101,110, - 116,114,94,0,0,0,114,27,0,0,0,114,23,0,0,0, - 114,201,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,198,0,0,0,65,3,0,0,115,38,0, - 0,0,0,2,18,1,6,2,22,1,18,1,17,2,19,1, - 15,1,3,1,17,1,13,2,7,1,18,3,16,1,27,1, - 3,1,16,1,17,1,18,2,122,25,83,111,117,114,99,101, - 70,105,108,101,76,111,97,100,101,114,46,115,101,116,95,100, - 97,116,97,78,41,7,114,112,0,0,0,114,111,0,0,0, - 114,113,0,0,0,114,114,0,0,0,114,197,0,0,0,114, - 199,0,0,0,114,198,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,218,0, - 0,0,51,3,0,0,115,8,0,0,0,12,2,6,2,12, - 5,12,5,114,218,0,0,0,99,0,0,0,0,0,0,0, - 0,0,0,0,0,2,0,0,0,64,0,0,0,115,46,0, - 0,0,101,0,0,90,1,0,100,0,0,90,2,0,100,1, - 0,90,3,0,100,2,0,100,3,0,132,0,0,90,4,0, - 100,4,0,100,5,0,132,0,0,90,5,0,100,6,0,83, - 41,7,218,20,83,111,117,114,99,101,108,101,115,115,70,105, - 108,101,76,111,97,100,101,114,122,45,76,111,97,100,101,114, - 32,119,104,105,99,104,32,104,97,110,100,108,101,115,32,115, - 111,117,114,99,101,108,101,115,115,32,102,105,108,101,32,105, - 109,112,111,114,116,115,46,99,2,0,0,0,0,0,0,0, - 5,0,0,0,6,0,0,0,67,0,0,0,115,76,0,0, - 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, - 124,0,0,106,1,0,124,2,0,131,1,0,125,3,0,116, - 2,0,124,3,0,100,1,0,124,1,0,100,2,0,124,2, - 0,131,1,2,125,4,0,116,3,0,124,4,0,100,1,0, - 124,1,0,100,3,0,124,2,0,131,1,2,83,41,4,78, - 114,106,0,0,0,114,35,0,0,0,114,89,0,0,0,41, - 4,114,157,0,0,0,114,200,0,0,0,114,141,0,0,0, - 114,147,0,0,0,41,5,114,108,0,0,0,114,126,0,0, - 0,114,35,0,0,0,114,53,0,0,0,114,209,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 187,0,0,0,98,3,0,0,115,8,0,0,0,0,1,15, - 1,15,1,24,1,122,29,83,111,117,114,99,101,108,101,115, - 115,70,105,108,101,76,111,97,100,101,114,46,103,101,116,95, - 99,111,100,101,99,2,0,0,0,0,0,0,0,2,0,0, + 97,100,101,114,46,105,115,95,112,97,99,107,97,103,101,32, + 98,121,32,99,104,101,99,107,105,110,103,32,105,102,10,32, + 32,32,32,32,32,32,32,116,104,101,32,112,97,116,104,32, + 114,101,116,117,114,110,101,100,32,98,121,32,103,101,116,95, + 102,105,108,101,110,97,109,101,32,104,97,115,32,97,32,102, + 105,108,101,110,97,109,101,32,111,102,32,39,95,95,105,110, + 105,116,95,95,46,112,121,39,46,114,29,0,0,0,114,58, + 0,0,0,114,59,0,0,0,114,56,0,0,0,218,8,95, + 95,105,110,105,116,95,95,41,4,114,38,0,0,0,114,151, + 0,0,0,114,34,0,0,0,114,32,0,0,0,41,5,114, + 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, + 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, + 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,153,0,0,0,125,2,0,0, + 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, + 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, + 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, + 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, + 0,100,1,0,83,41,2,122,42,85,115,101,32,100,101,102, + 97,117,108,116,32,115,101,109,97,110,116,105,99,115,32,102, + 111,114,32,109,111,100,117,108,101,32,99,114,101,97,116,105, + 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, + 117,108,101,133,2,0,0,115,0,0,0,0,122,27,95,76, + 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, + 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, + 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, + 0,0,0,124,0,0,106,0,0,124,1,0,106,1,0,131, + 1,0,125,2,0,124,2,0,100,1,0,107,8,0,114,54, + 0,116,2,0,100,2,0,106,3,0,124,1,0,106,1,0, + 131,1,0,131,1,0,130,1,0,116,4,0,106,5,0,116, + 6,0,124,2,0,124,1,0,106,7,0,131,3,0,1,100, + 1,0,83,41,3,122,19,69,120,101,99,117,116,101,32,116, + 104,101,32,109,111,100,117,108,101,46,78,122,52,99,97,110, + 110,111,116,32,108,111,97,100,32,109,111,100,117,108,101,32, + 123,33,114,125,32,119,104,101,110,32,103,101,116,95,99,111, + 100,101,40,41,32,114,101,116,117,114,110,115,32,78,111,110, + 101,41,8,218,8,103,101,116,95,99,111,100,101,114,105,0, + 0,0,114,99,0,0,0,114,47,0,0,0,114,114,0,0, + 0,218,25,95,99,97,108,108,95,119,105,116,104,95,102,114, + 97,109,101,115,95,114,101,109,111,118,101,100,218,4,101,120, + 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, + 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, + 95,109,111,100,117,108,101,136,2,0,0,115,10,0,0,0, + 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, + 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, + 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,3,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,106,1,0,124,0,0,124,1,0,131,2,0,83,41, + 1,78,41,2,114,114,0,0,0,218,17,95,108,111,97,100, + 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, + 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, + 100,117,108,101,144,2,0,0,115,2,0,0,0,0,1,122, + 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, + 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, + 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, + 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, + 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,178,0,0,0,120,2, + 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, + 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, + 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, + 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,100, + 2,0,132,0,0,90,3,0,100,3,0,100,4,0,132,0, + 0,90,4,0,100,5,0,100,6,0,132,0,0,90,5,0, + 100,7,0,100,8,0,132,0,0,90,6,0,100,9,0,100, + 10,0,132,0,0,90,7,0,100,11,0,100,18,0,100,13, + 0,100,14,0,132,0,1,90,8,0,100,15,0,100,16,0, + 132,0,0,90,9,0,100,17,0,83,41,19,218,12,83,111, + 117,114,99,101,76,111,97,100,101,114,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 10,0,0,0,116,0,0,130,1,0,100,1,0,83,41,2, + 122,178,79,112,116,105,111,110,97,108,32,109,101,116,104,111, + 100,32,116,104,97,116,32,114,101,116,117,114,110,115,32,116, + 104,101,32,109,111,100,105,102,105,99,97,116,105,111,110,32, + 116,105,109,101,32,40,97,110,32,105,110,116,41,32,102,111, + 114,32,116,104,101,10,32,32,32,32,32,32,32,32,115,112, + 101,99,105,102,105,101,100,32,112,97,116,104,44,32,119,104, + 101,114,101,32,112,97,116,104,32,105,115,32,97,32,115,116, + 114,46,10,10,32,32,32,32,32,32,32,32,82,97,105,115, + 101,115,32,73,79,69,114,114,111,114,32,119,104,101,110,32, + 116,104,101,32,112,97,116,104,32,99,97,110,110,111,116,32, + 98,101,32,104,97,110,100,108,101,100,46,10,32,32,32,32, + 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, + 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, + 104,95,109,116,105,109,101,150,2,0,0,115,2,0,0,0, + 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, + 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, + 115,19,0,0,0,100,1,0,124,0,0,106,0,0,124,1, + 0,131,1,0,105,1,0,83,41,2,97,170,1,0,0,79, + 112,116,105,111,110,97,108,32,109,101,116,104,111,100,32,114, + 101,116,117,114,110,105,110,103,32,97,32,109,101,116,97,100, + 97,116,97,32,100,105,99,116,32,102,111,114,32,116,104,101, + 32,115,112,101,99,105,102,105,101,100,32,112,97,116,104,10, + 32,32,32,32,32,32,32,32,116,111,32,98,121,32,116,104, + 101,32,112,97,116,104,32,40,115,116,114,41,46,10,32,32, + 32,32,32,32,32,32,80,111,115,115,105,98,108,101,32,107, + 101,121,115,58,10,32,32,32,32,32,32,32,32,45,32,39, + 109,116,105,109,101,39,32,40,109,97,110,100,97,116,111,114, + 121,41,32,105,115,32,116,104,101,32,110,117,109,101,114,105, + 99,32,116,105,109,101,115,116,97,109,112,32,111,102,32,108, + 97,115,116,32,115,111,117,114,99,101,10,32,32,32,32,32, + 32,32,32,32,32,99,111,100,101,32,109,111,100,105,102,105, + 99,97,116,105,111,110,59,10,32,32,32,32,32,32,32,32, + 45,32,39,115,105,122,101,39,32,40,111,112,116,105,111,110, + 97,108,41,32,105,115,32,116,104,101,32,115,105,122,101,32, + 105,110,32,98,121,116,101,115,32,111,102,32,116,104,101,32, + 115,111,117,114,99,101,32,99,111,100,101,46,10,10,32,32, + 32,32,32,32,32,32,73,109,112,108,101,109,101,110,116,105, + 110,103,32,116,104,105,115,32,109,101,116,104,111,100,32,97, + 108,108,111,119,115,32,116,104,101,32,108,111,97,100,101,114, + 32,116,111,32,114,101,97,100,32,98,121,116,101,99,111,100, + 101,32,102,105,108,101,115,46,10,32,32,32,32,32,32,32, + 32,82,97,105,115,101,115,32,73,79,69,114,114,111,114,32, + 119,104,101,110,32,116,104,101,32,112,97,116,104,32,99,97, + 110,110,111,116,32,98,101,32,104,97,110,100,108,101,100,46, + 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, + 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,10,112,97,116,104,95,115,116,97,116,115,158,2,0,0, + 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, + 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, + 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, + 0,67,0,0,0,115,16,0,0,0,124,0,0,106,0,0, + 124,2,0,124,3,0,131,2,0,83,41,1,122,228,79,112, + 116,105,111,110,97,108,32,109,101,116,104,111,100,32,119,104, + 105,99,104,32,119,114,105,116,101,115,32,100,97,116,97,32, + 40,98,121,116,101,115,41,32,116,111,32,97,32,102,105,108, + 101,32,112,97,116,104,32,40,97,32,115,116,114,41,46,10, + 10,32,32,32,32,32,32,32,32,73,109,112,108,101,109,101, + 110,116,105,110,103,32,116,104,105,115,32,109,101,116,104,111, + 100,32,97,108,108,111,119,115,32,102,111,114,32,116,104,101, + 32,119,114,105,116,105,110,103,32,111,102,32,98,121,116,101, + 99,111,100,101,32,102,105,108,101,115,46,10,10,32,32,32, + 32,32,32,32,32,84,104,101,32,115,111,117,114,99,101,32, + 112,97,116,104,32,105,115,32,110,101,101,100,101,100,32,105, + 110,32,111,114,100,101,114,32,116,111,32,99,111,114,114,101, + 99,116,108,121,32,116,114,97,110,115,102,101,114,32,112,101, + 114,109,105,115,115,105,111,110,115,10,32,32,32,32,32,32, + 32,32,41,1,218,8,115,101,116,95,100,97,116,97,41,4, + 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, + 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, + 104,101,95,98,121,116,101,99,111,100,101,171,2,0,0,115, + 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, + 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, + 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,39,82,101,116,117,114,110,32,78,111,110, - 101,32,97,115,32,116,104,101,114,101,32,105,115,32,110,111, - 32,115,111,117,114,99,101,32,99,111,100,101,46,78,114,4, - 0,0,0,41,2,114,108,0,0,0,114,126,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,202, - 0,0,0,104,3,0,0,115,2,0,0,0,0,2,122,31, - 83,111,117,114,99,101,108,101,115,115,70,105,108,101,76,111, - 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,78, - 41,6,114,112,0,0,0,114,111,0,0,0,114,113,0,0, - 0,114,114,0,0,0,114,187,0,0,0,114,202,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,223,0,0,0,94,3,0,0,115,6,0, - 0,0,12,2,6,2,12,6,114,223,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,3,0,0,0,64,0, - 0,0,115,136,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,100,2,0,100,3,0,132, - 0,0,90,4,0,100,4,0,100,5,0,132,0,0,90,5, - 0,100,6,0,100,7,0,132,0,0,90,6,0,100,8,0, - 100,9,0,132,0,0,90,7,0,100,10,0,100,11,0,132, - 0,0,90,8,0,100,12,0,100,13,0,132,0,0,90,9, - 0,100,14,0,100,15,0,132,0,0,90,10,0,100,16,0, - 100,17,0,132,0,0,90,11,0,101,12,0,100,18,0,100, - 19,0,132,0,0,131,1,0,90,13,0,100,20,0,83,41, - 21,218,19,69,120,116,101,110,115,105,111,110,70,105,108,101, - 76,111,97,100,101,114,122,93,76,111,97,100,101,114,32,102, - 111,114,32,101,120,116,101,110,115,105,111,110,32,109,111,100, - 117,108,101,115,46,10,10,32,32,32,32,84,104,101,32,99, - 111,110,115,116,114,117,99,116,111,114,32,105,115,32,100,101, - 115,105,103,110,101,100,32,116,111,32,119,111,114,107,32,119, - 105,116,104,32,70,105,108,101,70,105,110,100,101,114,46,10, - 10,32,32,32,32,99,3,0,0,0,0,0,0,0,3,0, - 0,0,2,0,0,0,67,0,0,0,115,22,0,0,0,124, - 1,0,124,0,0,95,0,0,124,2,0,124,0,0,95,1, - 0,100,0,0,83,41,1,78,41,2,114,106,0,0,0,114, - 35,0,0,0,41,3,114,108,0,0,0,114,106,0,0,0, - 114,35,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,185,0,0,0,121,3,0,0,115,4,0, - 0,0,0,1,9,1,122,28,69,120,116,101,110,115,105,111, - 110,70,105,108,101,76,111,97,100,101,114,46,95,95,105,110, - 105,116,95,95,99,2,0,0,0,0,0,0,0,2,0,0, - 0,2,0,0,0,67,0,0,0,115,34,0,0,0,124,0, - 0,106,0,0,124,1,0,106,0,0,107,2,0,111,33,0, - 124,0,0,106,1,0,124,1,0,106,1,0,107,2,0,83, - 41,1,78,41,2,114,211,0,0,0,114,118,0,0,0,41, - 2,114,108,0,0,0,114,212,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,213,0,0,0,125, - 3,0,0,115,4,0,0,0,0,1,18,1,122,26,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,101,113,95,95,99,1,0,0,0,0,0,0, - 0,1,0,0,0,3,0,0,0,67,0,0,0,115,26,0, - 0,0,116,0,0,124,0,0,106,1,0,131,1,0,116,0, - 0,124,0,0,106,2,0,131,1,0,65,83,41,1,78,41, - 3,114,214,0,0,0,114,106,0,0,0,114,35,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,215,0,0,0,129,3,0,0,115, - 2,0,0,0,0,1,122,28,69,120,116,101,110,115,105,111, - 110,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, - 115,104,95,95,99,2,0,0,0,0,0,0,0,3,0,0, - 0,4,0,0,0,67,0,0,0,115,47,0,0,0,116,0, - 0,106,1,0,116,2,0,106,3,0,124,1,0,131,2,0, - 125,2,0,116,4,0,100,1,0,124,1,0,106,5,0,124, - 0,0,106,6,0,131,3,0,1,124,2,0,83,41,2,122, - 38,67,114,101,97,116,101,32,97,110,32,117,110,105,116,105, - 97,108,105,122,101,100,32,101,120,116,101,110,115,105,111,110, - 32,109,111,100,117,108,101,122,38,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,32,123,33,114,125,32,108, - 111,97,100,101,100,32,102,114,111,109,32,123,33,114,125,41, - 7,114,121,0,0,0,114,188,0,0,0,114,145,0,0,0, - 90,14,99,114,101,97,116,101,95,100,121,110,97,109,105,99, - 114,105,0,0,0,114,106,0,0,0,114,35,0,0,0,41, - 3,114,108,0,0,0,114,164,0,0,0,114,190,0,0,0, + 0,83,41,2,122,150,79,112,116,105,111,110,97,108,32,109, + 101,116,104,111,100,32,119,104,105,99,104,32,119,114,105,116, + 101,115,32,100,97,116,97,32,40,98,121,116,101,115,41,32, + 116,111,32,97,32,102,105,108,101,32,112,97,116,104,32,40, + 97,32,115,116,114,41,46,10,10,32,32,32,32,32,32,32, + 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, + 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, + 32,102,111,114,32,116,104,101,32,119,114,105,116,105,110,103, + 32,111,102,32,98,121,116,101,99,111,100,101,32,102,105,108, + 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, + 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,114,192,0,0,0,181,2,0,0,115,0,0,0,0, + 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, + 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, + 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, + 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, + 121,19,0,124,0,0,106,1,0,124,2,0,131,1,0,125, + 3,0,87,110,58,0,4,116,2,0,107,10,0,114,94,0, + 1,125,4,0,1,122,26,0,116,3,0,100,1,0,100,2, + 0,124,1,0,131,1,1,124,4,0,130,2,0,87,89,100, + 3,0,100,3,0,125,4,0,126,4,0,88,110,1,0,88, + 116,4,0,124,3,0,131,1,0,83,41,4,122,52,67,111, + 110,99,114,101,116,101,32,105,109,112,108,101,109,101,110,116, + 97,116,105,111,110,32,111,102,32,73,110,115,112,101,99,116, + 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, + 101,46,122,39,115,111,117,114,99,101,32,110,111,116,32,97, + 118,97,105,108,97,98,108,101,32,116,104,114,111,117,103,104, + 32,103,101,116,95,100,97,116,97,40,41,114,98,0,0,0, + 78,41,5,114,151,0,0,0,218,8,103,101,116,95,100,97, + 116,97,114,40,0,0,0,114,99,0,0,0,114,149,0,0, + 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, + 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, + 95,115,111,117,114,99,101,188,2,0,0,115,14,0,0,0, + 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, + 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, + 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, + 122,101,114,29,0,0,0,99,3,0,0,0,1,0,0,0, + 4,0,0,0,9,0,0,0,67,0,0,0,115,34,0,0, + 0,116,0,0,106,1,0,116,2,0,124,1,0,124,2,0, + 100,1,0,100,2,0,100,3,0,100,4,0,124,3,0,131, + 4,2,83,41,5,122,130,82,101,116,117,114,110,32,116,104, + 101,32,99,111,100,101,32,111,98,106,101,99,116,32,99,111, + 109,112,105,108,101,100,32,102,114,111,109,32,115,111,117,114, + 99,101,46,10,10,32,32,32,32,32,32,32,32,84,104,101, + 32,39,100,97,116,97,39,32,97,114,103,117,109,101,110,116, + 32,99,97,110,32,98,101,32,97,110,121,32,111,98,106,101, + 99,116,32,116,121,112,101,32,116,104,97,116,32,99,111,109, + 112,105,108,101,40,41,32,115,117,112,112,111,114,116,115,46, + 10,32,32,32,32,32,32,32,32,114,183,0,0,0,218,12, + 100,111,110,116,95,105,110,104,101,114,105,116,84,114,68,0, + 0,0,41,3,114,114,0,0,0,114,182,0,0,0,218,7, + 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, + 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, + 117,114,99,101,95,116,111,95,99,111,100,101,198,2,0,0, + 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, + 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, + 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, + 0,0,0,43,0,0,0,67,0,0,0,115,183,1,0,0, + 124,0,0,106,0,0,124,1,0,131,1,0,125,2,0,100, + 1,0,125,3,0,121,16,0,116,1,0,124,2,0,131,1, + 0,125,4,0,87,110,24,0,4,116,2,0,107,10,0,114, + 63,0,1,1,1,100,1,0,125,4,0,89,110,205,0,88, + 121,19,0,124,0,0,106,3,0,124,2,0,131,1,0,125, + 5,0,87,110,18,0,4,116,4,0,107,10,0,114,103,0, + 1,1,1,89,110,165,0,88,116,5,0,124,5,0,100,2, + 0,25,131,1,0,125,3,0,121,19,0,124,0,0,106,6, + 0,124,4,0,131,1,0,125,6,0,87,110,18,0,4,116, + 7,0,107,10,0,114,159,0,1,1,1,89,110,109,0,88, + 121,34,0,116,8,0,124,6,0,100,3,0,124,5,0,100, + 4,0,124,1,0,100,5,0,124,4,0,131,1,3,125,7, + 0,87,110,24,0,4,116,9,0,116,10,0,102,2,0,107, + 10,0,114,220,0,1,1,1,89,110,48,0,88,116,11,0, + 106,12,0,100,6,0,124,4,0,124,2,0,131,3,0,1, + 116,13,0,124,7,0,100,4,0,124,1,0,100,7,0,124, + 4,0,100,8,0,124,2,0,131,1,3,83,124,0,0,106, + 6,0,124,2,0,131,1,0,125,8,0,124,0,0,106,14, + 0,124,8,0,124,2,0,131,2,0,125,9,0,116,11,0, + 106,12,0,100,9,0,124,2,0,131,2,0,1,116,15,0, + 106,16,0,12,114,179,1,124,4,0,100,1,0,107,9,0, + 114,179,1,124,3,0,100,1,0,107,9,0,114,179,1,116, + 17,0,124,9,0,124,3,0,116,18,0,124,8,0,131,1, + 0,131,3,0,125,6,0,121,39,0,124,0,0,106,19,0, + 124,2,0,124,4,0,124,6,0,131,3,0,1,116,11,0, + 106,12,0,100,10,0,124,4,0,131,2,0,1,87,110,18, + 0,4,116,2,0,107,10,0,114,178,1,1,1,1,89,110, + 1,0,88,124,9,0,83,41,11,122,190,67,111,110,99,114, + 101,116,101,32,105,109,112,108,101,109,101,110,116,97,116,105, + 111,110,32,111,102,32,73,110,115,112,101,99,116,76,111,97, + 100,101,114,46,103,101,116,95,99,111,100,101,46,10,10,32, + 32,32,32,32,32,32,32,82,101,97,100,105,110,103,32,111, + 102,32,98,121,116,101,99,111,100,101,32,114,101,113,117,105, + 114,101,115,32,112,97,116,104,95,115,116,97,116,115,32,116, + 111,32,98,101,32,105,109,112,108,101,109,101,110,116,101,100, + 46,32,84,111,32,119,114,105,116,101,10,32,32,32,32,32, + 32,32,32,98,121,116,101,99,111,100,101,44,32,115,101,116, + 95,100,97,116,97,32,109,117,115,116,32,97,108,115,111,32, + 98,101,32,105,109,112,108,101,109,101,110,116,101,100,46,10, + 10,32,32,32,32,32,32,32,32,78,114,126,0,0,0,114, + 132,0,0,0,114,98,0,0,0,114,35,0,0,0,122,13, + 123,125,32,109,97,116,99,104,101,115,32,123,125,114,89,0, + 0,0,114,90,0,0,0,122,19,99,111,100,101,32,111,98, + 106,101,99,116,32,102,114,111,109,32,123,125,122,10,119,114, + 111,116,101,32,123,33,114,125,41,20,114,151,0,0,0,114, + 79,0,0,0,114,66,0,0,0,114,191,0,0,0,114,189, + 0,0,0,114,14,0,0,0,114,194,0,0,0,114,40,0, + 0,0,114,135,0,0,0,114,99,0,0,0,114,130,0,0, + 0,114,114,0,0,0,114,129,0,0,0,114,141,0,0,0, + 114,200,0,0,0,114,7,0,0,0,218,19,100,111,110,116, + 95,119,114,105,116,101,95,98,121,116,101,99,111,100,101,114, + 144,0,0,0,114,31,0,0,0,114,193,0,0,0,41,10, + 114,100,0,0,0,114,119,0,0,0,114,90,0,0,0,114, + 133,0,0,0,114,89,0,0,0,218,2,115,116,114,53,0, + 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, + 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 186,0,0,0,132,3,0,0,115,10,0,0,0,0,2,6, - 1,15,1,6,1,16,1,122,33,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,99,114,101, - 97,116,101,95,109,111,100,117,108,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,4,0,0,0,67,0,0,0,115, - 45,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, - 124,1,0,131,2,0,1,116,4,0,100,1,0,124,0,0, - 106,5,0,124,0,0,106,6,0,131,3,0,1,100,2,0, - 83,41,3,122,30,73,110,105,116,105,97,108,105,122,101,32, - 97,110,32,101,120,116,101,110,115,105,111,110,32,109,111,100, - 117,108,101,122,40,101,120,116,101,110,115,105,111,110,32,109, - 111,100,117,108,101,32,123,33,114,125,32,101,120,101,99,117, - 116,101,100,32,102,114,111,109,32,123,33,114,125,78,41,7, - 114,121,0,0,0,114,188,0,0,0,114,145,0,0,0,90, - 12,101,120,101,99,95,100,121,110,97,109,105,99,114,105,0, - 0,0,114,106,0,0,0,114,35,0,0,0,41,2,114,108, - 0,0,0,114,190,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,191,0,0,0,140,3,0,0, - 115,6,0,0,0,0,2,19,1,6,1,122,31,69,120,116, - 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, - 46,101,120,101,99,95,109,111,100,117,108,101,99,2,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,3,0,0, - 0,115,48,0,0,0,116,0,0,124,0,0,106,1,0,131, - 1,0,100,1,0,25,137,0,0,116,2,0,135,0,0,102, - 1,0,100,2,0,100,3,0,134,0,0,116,3,0,68,131, - 1,0,131,1,0,83,41,4,122,49,82,101,116,117,114,110, - 32,84,114,117,101,32,105,102,32,116,104,101,32,101,120,116, - 101,110,115,105,111,110,32,109,111,100,117,108,101,32,105,115, - 32,97,32,112,97,99,107,97,103,101,46,114,29,0,0,0, - 99,1,0,0,0,0,0,0,0,2,0,0,0,4,0,0, - 0,51,0,0,0,115,31,0,0,0,124,0,0,93,21,0, - 125,1,0,136,0,0,100,0,0,124,1,0,23,107,2,0, - 86,1,113,3,0,100,1,0,83,41,2,114,185,0,0,0, - 78,114,4,0,0,0,41,2,114,22,0,0,0,218,6,115, - 117,102,102,105,120,41,1,218,9,102,105,108,101,95,110,97, - 109,101,114,4,0,0,0,114,5,0,0,0,250,9,60,103, - 101,110,101,120,112,114,62,149,3,0,0,115,2,0,0,0, - 6,1,122,49,69,120,116,101,110,115,105,111,110,70,105,108, - 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, - 103,101,46,60,108,111,99,97,108,115,62,46,60,103,101,110, - 101,120,112,114,62,41,4,114,38,0,0,0,114,35,0,0, - 0,218,3,97,110,121,218,18,69,88,84,69,78,83,73,79, - 78,95,83,85,70,70,73,88,69,83,41,2,114,108,0,0, - 0,114,126,0,0,0,114,4,0,0,0,41,1,114,226,0, - 0,0,114,5,0,0,0,114,159,0,0,0,146,3,0,0, - 115,6,0,0,0,0,2,19,1,18,1,122,30,69,120,116, - 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, - 46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,63,82,101,116, - 117,114,110,32,78,111,110,101,32,97,115,32,97,110,32,101, - 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,32, - 99,97,110,110,111,116,32,99,114,101,97,116,101,32,97,32, - 99,111,100,101,32,111,98,106,101,99,116,46,78,114,4,0, - 0,0,41,2,114,108,0,0,0,114,126,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,187,0, - 0,0,152,3,0,0,115,2,0,0,0,0,2,122,28,69, - 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, - 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,53,82,101,116, - 117,114,110,32,78,111,110,101,32,97,115,32,101,120,116,101, - 110,115,105,111,110,32,109,111,100,117,108,101,115,32,104,97, - 118,101,32,110,111,32,115,111,117,114,99,101,32,99,111,100, - 101,46,78,114,4,0,0,0,41,2,114,108,0,0,0,114, - 126,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,202,0,0,0,156,3,0,0,115,2,0,0, - 0,0,2,122,30,69,120,116,101,110,115,105,111,110,70,105, - 108,101,76,111,97,100,101,114,46,103,101,116,95,115,111,117, - 114,99,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 1,0,0,0,67,0,0,0,115,7,0,0,0,124,0,0, - 106,0,0,83,41,1,122,58,82,101,116,117,114,110,32,116, - 104,101,32,112,97,116,104,32,116,111,32,116,104,101,32,115, - 111,117,114,99,101,32,102,105,108,101,32,97,115,32,102,111, - 117,110,100,32,98,121,32,116,104,101,32,102,105,110,100,101, - 114,46,41,1,114,35,0,0,0,41,2,114,108,0,0,0, - 114,126,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,157,0,0,0,160,3,0,0,115,2,0, - 0,0,0,3,122,32,69,120,116,101,110,115,105,111,110,70, - 105,108,101,76,111,97,100,101,114,46,103,101,116,95,102,105, - 108,101,110,97,109,101,78,41,14,114,112,0,0,0,114,111, - 0,0,0,114,113,0,0,0,114,114,0,0,0,114,185,0, - 0,0,114,213,0,0,0,114,215,0,0,0,114,186,0,0, - 0,114,191,0,0,0,114,159,0,0,0,114,187,0,0,0, - 114,202,0,0,0,114,123,0,0,0,114,157,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,224,0,0,0,113,3,0,0,115,20,0,0, - 0,12,6,6,2,12,4,12,4,12,3,12,8,12,6,12, - 6,12,4,12,4,114,224,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,2,0,0,0,64,0,0,0,115, - 130,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, + 181,0,0,0,206,2,0,0,115,78,0,0,0,0,7,15, + 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, + 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, + 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, + 1,7,1,15,1,18,1,16,1,22,1,12,1,9,1,15, + 1,3,1,19,1,20,1,13,1,5,1,122,21,83,111,117, + 114,99,101,76,111,97,100,101,114,46,103,101,116,95,99,111, + 100,101,78,114,87,0,0,0,41,10,114,105,0,0,0,114, + 104,0,0,0,114,106,0,0,0,114,190,0,0,0,114,191, + 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, + 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,188,0,0,0,148,2,0,0,115,14,0,0,0,12,2, + 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, + 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, + 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, + 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, + 0,101,7,0,135,0,0,102,1,0,100,8,0,100,9,0, + 134,0,0,131,1,0,90,8,0,101,7,0,100,10,0,100, + 11,0,132,0,0,131,1,0,90,9,0,100,12,0,100,13, + 0,132,0,0,90,10,0,135,0,0,83,41,14,218,10,70, + 105,108,101,76,111,97,100,101,114,122,103,66,97,115,101,32, + 102,105,108,101,32,108,111,97,100,101,114,32,99,108,97,115, + 115,32,119,104,105,99,104,32,105,109,112,108,101,109,101,110, + 116,115,32,116,104,101,32,108,111,97,100,101,114,32,112,114, + 111,116,111,99,111,108,32,109,101,116,104,111,100,115,32,116, + 104,97,116,10,32,32,32,32,114,101,113,117,105,114,101,32, + 102,105,108,101,32,115,121,115,116,101,109,32,117,115,97,103, + 101,46,99,3,0,0,0,0,0,0,0,3,0,0,0,2, + 0,0,0,67,0,0,0,115,22,0,0,0,124,1,0,124, + 0,0,95,0,0,124,2,0,124,0,0,95,1,0,100,1, + 0,83,41,2,122,75,67,97,99,104,101,32,116,104,101,32, + 109,111,100,117,108,101,32,110,97,109,101,32,97,110,100,32, + 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, + 102,105,108,101,32,102,111,117,110,100,32,98,121,32,116,104, + 101,10,32,32,32,32,32,32,32,32,102,105,110,100,101,114, + 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, + 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, + 0,0,0,7,3,0,0,115,4,0,0,0,0,3,9,1, + 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, + 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, + 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, + 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, + 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, + 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, + 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, + 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,6,95,95,101,113,95,95,13,3,0,0,115, + 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, + 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, + 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, + 115,26,0,0,0,116,0,0,124,0,0,106,1,0,131,1, + 0,116,0,0,124,0,0,106,2,0,131,1,0,65,83,41, + 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, + 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, + 115,104,95,95,17,3,0,0,115,2,0,0,0,0,1,122, + 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, + 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, + 0,116,1,0,124,0,0,131,2,0,106,2,0,124,1,0, + 131,1,0,83,41,1,122,100,76,111,97,100,32,97,32,109, + 111,100,117,108,101,32,102,114,111,109,32,97,32,102,105,108, + 101,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, + 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, + 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, + 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, + 100,46,10,10,32,32,32,32,32,32,32,32,41,3,218,5, + 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, + 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, + 0,20,3,0,0,115,2,0,0,0,0,10,122,22,70,105, + 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, + 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, + 0,106,0,0,83,41,1,122,58,82,101,116,117,114,110,32, + 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, + 115,111,117,114,99,101,32,102,105,108,101,32,97,115,32,102, + 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, + 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,151,0,0,0,32,3,0,0,115,2, + 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, + 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, + 0,0,0,115,42,0,0,0,116,0,0,106,1,0,124,1, + 0,100,1,0,131,2,0,143,17,0,125,2,0,124,2,0, + 106,2,0,131,0,0,83,87,100,2,0,81,82,88,100,2, + 0,83,41,3,122,39,82,101,116,117,114,110,32,116,104,101, + 32,100,97,116,97,32,102,114,111,109,32,112,97,116,104,32, + 97,115,32,114,97,119,32,98,121,116,101,115,46,218,1,114, + 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, + 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, + 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,194,0,0,0,37,3,0,0,115,4,0,0, + 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, + 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, + 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, + 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, + 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, + 205,0,0,0,114,5,0,0,0,114,204,0,0,0,2,3, + 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, + 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, + 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, + 64,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,132,0,0,90,6,0,100,8,0,100,9,0, - 132,0,0,90,7,0,100,10,0,100,11,0,132,0,0,90, - 8,0,100,12,0,100,13,0,132,0,0,90,9,0,100,14, - 0,100,15,0,132,0,0,90,10,0,100,16,0,100,17,0, - 132,0,0,90,11,0,100,18,0,100,19,0,132,0,0,90, - 12,0,100,20,0,83,41,21,218,14,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,97,38,1,0,0,82,101,112, - 114,101,115,101,110,116,115,32,97,32,110,97,109,101,115,112, - 97,99,101,32,112,97,99,107,97,103,101,39,115,32,112,97, - 116,104,46,32,32,73,116,32,117,115,101,115,32,116,104,101, - 32,109,111,100,117,108,101,32,110,97,109,101,10,32,32,32, - 32,116,111,32,102,105,110,100,32,105,116,115,32,112,97,114, - 101,110,116,32,109,111,100,117,108,101,44,32,97,110,100,32, - 102,114,111,109,32,116,104,101,114,101,32,105,116,32,108,111, - 111,107,115,32,117,112,32,116,104,101,32,112,97,114,101,110, - 116,39,115,10,32,32,32,32,95,95,112,97,116,104,95,95, - 46,32,32,87,104,101,110,32,116,104,105,115,32,99,104,97, - 110,103,101,115,44,32,116,104,101,32,109,111,100,117,108,101, - 39,115,32,111,119,110,32,112,97,116,104,32,105,115,32,114, - 101,99,111,109,112,117,116,101,100,44,10,32,32,32,32,117, - 115,105,110,103,32,112,97,116,104,95,102,105,110,100,101,114, - 46,32,32,70,111,114,32,116,111,112,45,108,101,118,101,108, - 32,109,111,100,117,108,101,115,44,32,116,104,101,32,112,97, - 114,101,110,116,32,109,111,100,117,108,101,39,115,32,112,97, - 116,104,10,32,32,32,32,105,115,32,115,121,115,46,112,97, - 116,104,46,99,4,0,0,0,0,0,0,0,4,0,0,0, - 2,0,0,0,67,0,0,0,115,52,0,0,0,124,1,0, - 124,0,0,95,0,0,124,2,0,124,0,0,95,1,0,116, - 2,0,124,0,0,106,3,0,131,0,0,131,1,0,124,0, - 0,95,4,0,124,3,0,124,0,0,95,5,0,100,0,0, - 83,41,1,78,41,6,218,5,95,110,97,109,101,218,5,95, - 112,97,116,104,114,93,0,0,0,218,16,95,103,101,116,95, - 112,97,114,101,110,116,95,112,97,116,104,218,17,95,108,97, - 115,116,95,112,97,114,101,110,116,95,112,97,116,104,218,12, - 95,112,97,116,104,95,102,105,110,100,101,114,41,4,114,108, - 0,0,0,114,106,0,0,0,114,35,0,0,0,218,11,112, - 97,116,104,95,102,105,110,100,101,114,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,185,0,0,0,173,3, - 0,0,115,8,0,0,0,0,1,9,1,9,1,21,1,122, - 23,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 95,95,105,110,105,116,95,95,99,1,0,0,0,0,0,0, - 0,4,0,0,0,3,0,0,0,67,0,0,0,115,53,0, - 0,0,124,0,0,106,0,0,106,1,0,100,1,0,131,1, - 0,92,3,0,125,1,0,125,2,0,125,3,0,124,2,0, - 100,2,0,107,2,0,114,43,0,100,6,0,83,124,1,0, - 100,5,0,102,2,0,83,41,7,122,62,82,101,116,117,114, - 110,115,32,97,32,116,117,112,108,101,32,111,102,32,40,112, - 97,114,101,110,116,45,109,111,100,117,108,101,45,110,97,109, - 101,44,32,112,97,114,101,110,116,45,112,97,116,104,45,97, - 116,116,114,45,110,97,109,101,41,114,58,0,0,0,114,30, - 0,0,0,114,7,0,0,0,114,35,0,0,0,90,8,95, - 95,112,97,116,104,95,95,41,2,122,3,115,121,115,122,4, - 112,97,116,104,41,2,114,231,0,0,0,114,32,0,0,0, - 41,4,114,108,0,0,0,114,222,0,0,0,218,3,100,111, - 116,90,2,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,23,95,102,105,110,100,95,112,97,114,101, - 110,116,95,112,97,116,104,95,110,97,109,101,115,179,3,0, - 0,115,8,0,0,0,0,2,27,1,12,2,4,3,122,38, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 102,105,110,100,95,112,97,114,101,110,116,95,112,97,116,104, - 95,110,97,109,101,115,99,1,0,0,0,0,0,0,0,3, - 0,0,0,3,0,0,0,67,0,0,0,115,38,0,0,0, - 124,0,0,106,0,0,131,0,0,92,2,0,125,1,0,125, - 2,0,116,1,0,116,2,0,106,3,0,124,1,0,25,124, - 2,0,131,2,0,83,41,1,78,41,4,114,238,0,0,0, - 114,117,0,0,0,114,7,0,0,0,218,7,109,111,100,117, - 108,101,115,41,3,114,108,0,0,0,90,18,112,97,114,101, - 110,116,95,109,111,100,117,108,101,95,110,97,109,101,90,14, - 112,97,116,104,95,97,116,116,114,95,110,97,109,101,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,233,0, - 0,0,189,3,0,0,115,4,0,0,0,0,1,18,1,122, - 31,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 95,103,101,116,95,112,97,114,101,110,116,95,112,97,116,104, - 99,1,0,0,0,0,0,0,0,3,0,0,0,3,0,0, - 0,67,0,0,0,115,118,0,0,0,116,0,0,124,0,0, - 106,1,0,131,0,0,131,1,0,125,1,0,124,1,0,124, - 0,0,106,2,0,107,3,0,114,111,0,124,0,0,106,3, - 0,124,0,0,106,4,0,124,1,0,131,2,0,125,2,0, - 124,2,0,100,0,0,107,9,0,114,102,0,124,2,0,106, - 5,0,100,0,0,107,8,0,114,102,0,124,2,0,106,6, - 0,114,102,0,124,2,0,106,6,0,124,0,0,95,7,0, - 124,1,0,124,0,0,95,2,0,124,0,0,106,7,0,83, - 41,1,78,41,8,114,93,0,0,0,114,233,0,0,0,114, - 234,0,0,0,114,235,0,0,0,114,231,0,0,0,114,127, - 0,0,0,114,156,0,0,0,114,232,0,0,0,41,3,114, - 108,0,0,0,90,11,112,97,114,101,110,116,95,112,97,116, - 104,114,164,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,12,95,114,101,99,97,108,99,117,108, - 97,116,101,193,3,0,0,115,16,0,0,0,0,2,18,1, - 15,1,21,3,27,1,9,1,12,1,9,1,122,27,95,78, - 97,109,101,115,112,97,99,101,80,97,116,104,46,95,114,101, - 99,97,108,99,117,108,97,116,101,99,1,0,0,0,0,0, - 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, - 0,0,0,116,0,0,124,0,0,106,1,0,131,0,0,131, - 1,0,83,41,1,78,41,2,218,4,105,116,101,114,114,240, - 0,0,0,41,1,114,108,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,8,95,95,105,116,101, - 114,95,95,206,3,0,0,115,2,0,0,0,0,1,122,23, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 95,105,116,101,114,95,95,99,1,0,0,0,0,0,0,0, - 1,0,0,0,2,0,0,0,67,0,0,0,115,16,0,0, - 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, - 83,41,1,78,41,2,114,31,0,0,0,114,240,0,0,0, - 41,1,114,108,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,7,95,95,108,101,110,95,95,209, - 3,0,0,115,2,0,0,0,0,1,122,22,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,108,101,110, - 95,95,99,1,0,0,0,0,0,0,0,1,0,0,0,2, - 0,0,0,67,0,0,0,115,16,0,0,0,100,1,0,106, - 0,0,124,0,0,106,1,0,131,1,0,83,41,2,78,122, - 20,95,78,97,109,101,115,112,97,99,101,80,97,116,104,40, - 123,33,114,125,41,41,2,114,47,0,0,0,114,232,0,0, - 0,41,1,114,108,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,8,95,95,114,101,112,114,95, - 95,212,3,0,0,115,2,0,0,0,0,1,122,23,95,78, - 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,114, - 101,112,114,95,95,99,2,0,0,0,0,0,0,0,2,0, - 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,124, - 1,0,124,0,0,106,0,0,131,0,0,107,6,0,83,41, - 1,78,41,1,114,240,0,0,0,41,2,114,108,0,0,0, - 218,4,105,116,101,109,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,12,95,95,99,111,110,116,97,105,110, - 115,95,95,215,3,0,0,115,2,0,0,0,0,1,122,27, - 95,78,97,109,101,115,112,97,99,101,80,97,116,104,46,95, - 95,99,111,110,116,97,105,110,115,95,95,99,2,0,0,0, - 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, - 115,20,0,0,0,124,0,0,106,0,0,106,1,0,124,1, - 0,131,1,0,1,100,0,0,83,41,1,78,41,2,114,232, - 0,0,0,114,163,0,0,0,41,2,114,108,0,0,0,114, - 245,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,163,0,0,0,218,3,0,0,115,2,0,0, - 0,0,1,122,21,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,97,112,112,101,110,100,78,41,13,114,112,0, - 0,0,114,111,0,0,0,114,113,0,0,0,114,114,0,0, - 0,114,185,0,0,0,114,238,0,0,0,114,233,0,0,0, - 114,240,0,0,0,114,242,0,0,0,114,243,0,0,0,114, - 244,0,0,0,114,246,0,0,0,114,163,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,230,0,0,0,166,3,0,0,115,20,0,0,0, - 12,5,6,2,12,6,12,10,12,4,12,13,12,3,12,3, - 12,3,12,3,114,230,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,118, - 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,100,2,0,132,0,0,90,3,0,101,4,0,100,3, - 0,100,4,0,132,0,0,131,1,0,90,5,0,100,5,0, - 100,6,0,132,0,0,90,6,0,100,7,0,100,8,0,132, - 0,0,90,7,0,100,9,0,100,10,0,132,0,0,90,8, - 0,100,11,0,100,12,0,132,0,0,90,9,0,100,13,0, - 100,14,0,132,0,0,90,10,0,100,15,0,100,16,0,132, - 0,0,90,11,0,100,17,0,83,41,18,218,16,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,99,4,0, - 0,0,0,0,0,0,4,0,0,0,4,0,0,0,67,0, - 0,0,115,25,0,0,0,116,0,0,124,1,0,124,2,0, - 124,3,0,131,3,0,124,0,0,95,1,0,100,0,0,83, - 41,1,78,41,2,114,230,0,0,0,114,232,0,0,0,41, - 4,114,108,0,0,0,114,106,0,0,0,114,35,0,0,0, - 114,236,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,185,0,0,0,224,3,0,0,115,2,0, - 0,0,0,1,122,25,95,78,97,109,101,115,112,97,99,101, - 76,111,97,100,101,114,46,95,95,105,110,105,116,95,95,99, - 2,0,0,0,0,0,0,0,2,0,0,0,2,0,0,0, - 67,0,0,0,115,16,0,0,0,100,1,0,106,0,0,124, - 1,0,106,1,0,131,1,0,83,41,2,122,115,82,101,116, - 117,114,110,32,114,101,112,114,32,102,111,114,32,116,104,101, - 32,109,111,100,117,108,101,46,10,10,32,32,32,32,32,32, - 32,32,84,104,101,32,109,101,116,104,111,100,32,105,115,32, - 100,101,112,114,101,99,97,116,101,100,46,32,32,84,104,101, - 32,105,109,112,111,114,116,32,109,97,99,104,105,110,101,114, - 121,32,100,111,101,115,32,116,104,101,32,106,111,98,32,105, - 116,115,101,108,102,46,10,10,32,32,32,32,32,32,32,32, - 122,25,60,109,111,100,117,108,101,32,123,33,114,125,32,40, - 110,97,109,101,115,112,97,99,101,41,62,41,2,114,47,0, - 0,0,114,112,0,0,0,41,2,114,170,0,0,0,114,190, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,11,109,111,100,117,108,101,95,114,101,112,114,227, - 3,0,0,115,2,0,0,0,0,7,122,28,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,109,111,100, - 117,108,101,95,114,101,112,114,99,2,0,0,0,0,0,0, - 0,2,0,0,0,1,0,0,0,67,0,0,0,115,4,0, - 0,0,100,1,0,83,41,2,78,84,114,4,0,0,0,41, - 2,114,108,0,0,0,114,126,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,159,0,0,0,236, - 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,105,115,95, + 0,100,7,0,100,8,0,100,9,0,132,0,1,90,6,0, + 100,10,0,83,41,11,218,16,83,111,117,114,99,101,70,105, + 108,101,76,111,97,100,101,114,122,62,67,111,110,99,114,101, + 116,101,32,105,109,112,108,101,109,101,110,116,97,116,105,111, + 110,32,111,102,32,83,111,117,114,99,101,76,111,97,100,101, + 114,32,117,115,105,110,103,32,116,104,101,32,102,105,108,101, + 32,115,121,115,116,101,109,46,99,2,0,0,0,0,0,0, + 0,3,0,0,0,4,0,0,0,67,0,0,0,115,34,0, + 0,0,116,0,0,124,1,0,131,1,0,125,2,0,100,1, + 0,124,2,0,106,1,0,100,2,0,124,2,0,106,2,0, + 105,2,0,83,41,3,122,33,82,101,116,117,114,110,32,116, + 104,101,32,109,101,116,97,100,97,116,97,32,102,111,114,32, + 116,104,101,32,112,97,116,104,46,114,126,0,0,0,114,127, + 0,0,0,41,3,114,39,0,0,0,218,8,115,116,95,109, + 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, + 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, + 0,0,47,3,0,0,115,4,0,0,0,0,2,12,1,122, + 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, + 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, + 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, + 0,115,34,0,0,0,116,0,0,124,1,0,131,1,0,125, + 4,0,124,0,0,106,1,0,124,2,0,124,3,0,100,1, + 0,124,4,0,131,2,1,83,41,2,78,218,5,95,109,111, + 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, + 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, + 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,193,0,0,0,52,3,0, + 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, + 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, + 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, + 0,105,182,1,0,0,99,3,0,0,0,1,0,0,0,9, + 0,0,0,17,0,0,0,67,0,0,0,115,62,1,0,0, + 116,0,0,124,1,0,131,1,0,92,2,0,125,4,0,125, + 5,0,103,0,0,125,6,0,120,54,0,124,4,0,114,80, + 0,116,1,0,124,4,0,131,1,0,12,114,80,0,116,0, + 0,124,4,0,131,1,0,92,2,0,125,4,0,125,7,0, + 124,6,0,106,2,0,124,7,0,131,1,0,1,113,27,0, + 87,120,135,0,116,3,0,124,6,0,131,1,0,68,93,121, + 0,125,7,0,116,4,0,124,4,0,124,7,0,131,2,0, + 125,4,0,121,17,0,116,5,0,106,6,0,124,4,0,131, + 1,0,1,87,113,94,0,4,116,7,0,107,10,0,114,155, + 0,1,1,1,119,94,0,89,113,94,0,4,116,8,0,107, + 10,0,114,214,0,1,125,8,0,1,122,28,0,116,9,0, + 106,10,0,100,1,0,124,4,0,124,8,0,131,3,0,1, + 100,2,0,83,87,89,100,2,0,100,2,0,125,8,0,126, + 8,0,88,113,94,0,88,113,94,0,87,121,36,0,116,11, + 0,124,1,0,124,2,0,124,3,0,131,3,0,1,116,9, + 0,106,10,0,100,3,0,124,1,0,131,2,0,1,87,110, + 56,0,4,116,8,0,107,10,0,114,57,1,1,125,8,0, + 1,122,24,0,116,9,0,106,10,0,100,1,0,124,1,0, + 124,8,0,131,3,0,1,87,89,100,2,0,100,2,0,125, + 8,0,126,8,0,88,110,1,0,88,100,2,0,83,41,4, + 122,27,87,114,105,116,101,32,98,121,116,101,115,32,100,97, + 116,97,32,116,111,32,97,32,102,105,108,101,46,122,27,99, + 111,117,108,100,32,110,111,116,32,99,114,101,97,116,101,32, + 123,33,114,125,58,32,123,33,114,125,78,122,12,99,114,101, + 97,116,101,100,32,123,33,114,125,41,12,114,38,0,0,0, + 114,46,0,0,0,114,157,0,0,0,114,33,0,0,0,114, + 28,0,0,0,114,3,0,0,0,90,5,109,107,100,105,114, + 218,15,70,105,108,101,69,120,105,115,116,115,69,114,114,111, + 114,114,40,0,0,0,114,114,0,0,0,114,129,0,0,0, + 114,55,0,0,0,41,9,114,100,0,0,0,114,35,0,0, + 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, + 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, + 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,192,0,0,0,57,3,0,0,115, + 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, + 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, + 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, + 83,111,117,114,99,101,70,105,108,101,76,111,97,100,101,114, + 46,115,101,116,95,100,97,116,97,78,41,7,114,105,0,0, + 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, + 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,212,0,0,0,43,3,0,0,115,8,0,0, + 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, + 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, + 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, + 5,0,100,6,0,83,41,7,218,20,83,111,117,114,99,101, + 108,101,115,115,70,105,108,101,76,111,97,100,101,114,122,45, + 76,111,97,100,101,114,32,119,104,105,99,104,32,104,97,110, + 100,108,101,115,32,115,111,117,114,99,101,108,101,115,115,32, + 102,105,108,101,32,105,109,112,111,114,116,115,46,99,2,0, + 0,0,0,0,0,0,5,0,0,0,6,0,0,0,67,0, + 0,0,115,76,0,0,0,124,0,0,106,0,0,124,1,0, + 131,1,0,125,2,0,124,0,0,106,1,0,124,2,0,131, + 1,0,125,3,0,116,2,0,124,3,0,100,1,0,124,1, + 0,100,2,0,124,2,0,131,1,2,125,4,0,116,3,0, + 124,4,0,100,1,0,124,1,0,100,3,0,124,2,0,131, + 1,2,83,41,4,78,114,98,0,0,0,114,35,0,0,0, + 114,89,0,0,0,41,4,114,151,0,0,0,114,194,0,0, + 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, + 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,181,0,0,0,92,3,0,0,115,8, + 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, + 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, + 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 4,0,0,0,100,1,0,83,41,2,122,39,82,101,116,117, + 114,110,32,78,111,110,101,32,97,115,32,116,104,101,114,101, + 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, + 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,196,0,0,0,98,3,0,0,115,2,0, + 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, + 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, + 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, + 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,88, + 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, + 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, + 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, + 90,1,0,100,0,0,90,2,0,100,1,0,90,3,0,100, + 2,0,100,3,0,132,0,0,90,4,0,100,4,0,100,5, + 0,132,0,0,90,5,0,100,6,0,100,7,0,132,0,0, + 90,6,0,100,8,0,100,9,0,132,0,0,90,7,0,100, + 10,0,100,11,0,132,0,0,90,8,0,100,12,0,100,13, + 0,132,0,0,90,9,0,100,14,0,100,15,0,132,0,0, + 90,10,0,100,16,0,100,17,0,132,0,0,90,11,0,101, + 12,0,100,18,0,100,19,0,132,0,0,131,1,0,90,13, + 0,100,20,0,83,41,21,218,19,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,122,93,76,111, + 97,100,101,114,32,102,111,114,32,101,120,116,101,110,115,105, + 111,110,32,109,111,100,117,108,101,115,46,10,10,32,32,32, + 32,84,104,101,32,99,111,110,115,116,114,117,99,116,111,114, + 32,105,115,32,100,101,115,105,103,110,101,100,32,116,111,32, + 119,111,114,107,32,119,105,116,104,32,70,105,108,101,70,105, + 110,100,101,114,46,10,10,32,32,32,32,99,3,0,0,0, + 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, + 115,22,0,0,0,124,1,0,124,0,0,95,0,0,124,2, + 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, + 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, + 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,115, + 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, + 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, + 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, + 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, + 34,0,0,0,124,0,0,106,0,0,124,1,0,106,0,0, + 107,2,0,111,33,0,124,0,0,106,1,0,124,1,0,106, + 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, + 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,207,0,0,0,119,3,0,0,115,4,0,0,0,0,1, + 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, + 0,0,0,115,26,0,0,0,116,0,0,124,0,0,106,1, + 0,131,1,0,116,0,0,124,0,0,106,2,0,131,1,0, + 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, + 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, + 0,123,3,0,0,115,2,0,0,0,0,1,122,28,69,120, + 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, + 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, + 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, + 50,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, + 124,1,0,131,2,0,125,2,0,116,0,0,106,4,0,100, + 1,0,124,1,0,106,5,0,124,0,0,106,6,0,131,3, + 0,1,124,2,0,83,41,2,122,38,67,114,101,97,116,101, + 32,97,110,32,117,110,105,116,105,97,108,105,122,101,100,32, + 101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,101, + 122,38,101,120,116,101,110,115,105,111,110,32,109,111,100,117, + 108,101,32,123,33,114,125,32,108,111,97,100,101,100,32,102, + 114,111,109,32,123,33,114,125,41,7,114,114,0,0,0,114, + 182,0,0,0,114,139,0,0,0,90,14,99,114,101,97,116, + 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, + 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, + 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,180,0,0,0,126,3,0, + 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, + 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, + 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 4,0,0,0,67,0,0,0,115,48,0,0,0,116,0,0, + 106,1,0,116,2,0,106,3,0,124,1,0,131,2,0,1, + 116,0,0,106,4,0,100,1,0,124,0,0,106,5,0,124, + 0,0,106,6,0,131,3,0,1,100,2,0,83,41,3,122, + 30,73,110,105,116,105,97,108,105,122,101,32,97,110,32,101, + 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,122, + 40,101,120,116,101,110,115,105,111,110,32,109,111,100,117,108, + 101,32,123,33,114,125,32,101,120,101,99,117,116,101,100,32, + 102,114,111,109,32,123,33,114,125,78,41,7,114,114,0,0, + 0,114,182,0,0,0,114,139,0,0,0,90,12,101,120,101, + 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, + 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, + 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,185,0,0,0,134,3,0,0,115,6,0,0, + 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, + 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, + 0,2,0,0,0,4,0,0,0,3,0,0,0,115,48,0, + 0,0,116,0,0,124,0,0,106,1,0,131,1,0,100,1, + 0,25,137,0,0,116,2,0,135,0,0,102,1,0,100,2, + 0,100,3,0,134,0,0,116,3,0,68,131,1,0,131,1, + 0,83,41,4,122,49,82,101,116,117,114,110,32,84,114,117, + 101,32,105,102,32,116,104,101,32,101,120,116,101,110,115,105, + 111,110,32,109,111,100,117,108,101,32,105,115,32,97,32,112, + 97,99,107,97,103,101,46,114,29,0,0,0,99,1,0,0, + 0,0,0,0,0,2,0,0,0,4,0,0,0,51,0,0, + 0,115,31,0,0,0,124,0,0,93,21,0,125,1,0,136, + 0,0,100,0,0,124,1,0,23,107,2,0,86,1,113,3, + 0,100,1,0,83,41,2,114,179,0,0,0,78,114,4,0, + 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, + 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, + 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, + 112,114,62,143,3,0,0,115,2,0,0,0,6,1,122,49, + 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, + 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, + 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, + 62,41,4,114,38,0,0,0,114,35,0,0,0,218,3,97, + 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, + 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, + 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, + 0,0,0,114,153,0,0,0,140,3,0,0,115,6,0,0, + 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,78,114,30,0,0,0,114,4,0, - 0,0,41,2,114,108,0,0,0,114,126,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,202,0, - 0,0,239,3,0,0,115,2,0,0,0,0,1,122,27,95, - 78,97,109,101,115,112,97,99,101,76,111,97,100,101,114,46, - 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,6,0,0,0,67,0,0,0,115, - 22,0,0,0,116,0,0,100,1,0,100,2,0,100,3,0, - 100,4,0,100,5,0,131,3,1,83,41,6,78,114,30,0, - 0,0,122,8,60,115,116,114,105,110,103,62,114,189,0,0, - 0,114,204,0,0,0,84,41,1,114,205,0,0,0,41,2, - 114,108,0,0,0,114,126,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,187,0,0,0,242,3, - 0,0,115,2,0,0,0,0,1,122,25,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, - 99,111,100,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,42,85,115,101,32,100,101,102,97,117,108, - 116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,32, - 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, - 78,114,4,0,0,0,41,2,114,108,0,0,0,114,164,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,186,0,0,0,245,3,0,0,115,0,0,0,0,122, - 30,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, - 114,46,99,114,101,97,116,101,95,109,111,100,117,108,101,99, + 0,100,1,0,83,41,2,122,63,82,101,116,117,114,110,32, + 78,111,110,101,32,97,115,32,97,110,32,101,120,116,101,110, + 115,105,111,110,32,109,111,100,117,108,101,32,99,97,110,110, + 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, + 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, + 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,181,0,0,0,146,3, + 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, + 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, + 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, + 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, + 0,100,1,0,83,41,2,122,53,82,101,116,117,114,110,32, + 78,111,110,101,32,97,115,32,101,120,116,101,110,115,105,111, + 110,32,109,111,100,117,108,101,115,32,104,97,118,101,32,110, + 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 196,0,0,0,150,3,0,0,115,2,0,0,0,0,2,122, + 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, + 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, - 67,0,0,0,115,4,0,0,0,100,0,0,83,41,1,78, - 114,4,0,0,0,41,2,114,108,0,0,0,114,190,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,191,0,0,0,248,3,0,0,115,2,0,0,0,0,1, - 122,28,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,101,120,101,99,95,109,111,100,117,108,101,99,2, - 0,0,0,0,0,0,0,2,0,0,0,3,0,0,0,67, - 0,0,0,115,32,0,0,0,116,0,0,100,1,0,124,0, - 0,106,1,0,131,2,0,1,116,2,0,106,3,0,124,0, - 0,124,1,0,131,2,0,83,41,2,122,98,76,111,97,100, - 32,97,32,110,97,109,101,115,112,97,99,101,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, - 105,115,32,109,101,116,104,111,100,32,105,115,32,100,101,112, - 114,101,99,97,116,101,100,46,32,32,85,115,101,32,101,120, - 101,99,95,109,111,100,117,108,101,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,122,38, - 110,97,109,101,115,112,97,99,101,32,109,111,100,117,108,101, - 32,108,111,97,100,101,100,32,119,105,116,104,32,112,97,116, - 104,32,123,33,114,125,41,4,114,105,0,0,0,114,232,0, - 0,0,114,121,0,0,0,114,192,0,0,0,41,2,114,108, - 0,0,0,114,126,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,193,0,0,0,251,3,0,0, - 115,4,0,0,0,0,7,16,1,122,28,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,108,111,97,100, - 95,109,111,100,117,108,101,78,41,12,114,112,0,0,0,114, - 111,0,0,0,114,113,0,0,0,114,185,0,0,0,114,183, - 0,0,0,114,248,0,0,0,114,159,0,0,0,114,202,0, - 0,0,114,187,0,0,0,114,186,0,0,0,114,191,0,0, - 0,114,193,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,247,0,0,0,223, - 3,0,0,115,16,0,0,0,12,1,12,3,18,9,12,3, - 12,3,12,3,12,3,12,3,114,247,0,0,0,99,0,0, - 0,0,0,0,0,0,0,0,0,0,5,0,0,0,64,0, - 0,0,115,160,0,0,0,101,0,0,90,1,0,100,0,0, - 90,2,0,100,1,0,90,3,0,101,4,0,100,2,0,100, - 3,0,132,0,0,131,1,0,90,5,0,101,4,0,100,4, - 0,100,5,0,132,0,0,131,1,0,90,6,0,101,4,0, - 100,6,0,100,7,0,132,0,0,131,1,0,90,7,0,101, - 4,0,100,8,0,100,9,0,132,0,0,131,1,0,90,8, - 0,101,4,0,100,10,0,100,11,0,100,12,0,132,1,0, - 131,1,0,90,9,0,101,4,0,100,10,0,100,10,0,100, - 13,0,100,14,0,132,2,0,131,1,0,90,10,0,101,4, - 0,100,10,0,100,15,0,100,16,0,132,1,0,131,1,0, - 90,11,0,100,10,0,83,41,17,218,10,80,97,116,104,70, - 105,110,100,101,114,122,62,77,101,116,97,32,112,97,116,104, - 32,102,105,110,100,101,114,32,102,111,114,32,115,121,115,46, - 112,97,116,104,32,97,110,100,32,112,97,99,107,97,103,101, - 32,95,95,112,97,116,104,95,95,32,97,116,116,114,105,98, - 117,116,101,115,46,99,1,0,0,0,0,0,0,0,2,0, - 0,0,4,0,0,0,67,0,0,0,115,55,0,0,0,120, - 48,0,116,0,0,106,1,0,106,2,0,131,0,0,68,93, - 31,0,125,1,0,116,3,0,124,1,0,100,1,0,131,2, - 0,114,16,0,124,1,0,106,4,0,131,0,0,1,113,16, - 0,87,100,2,0,83,41,3,122,125,67,97,108,108,32,116, - 104,101,32,105,110,118,97,108,105,100,97,116,101,95,99,97, - 99,104,101,115,40,41,32,109,101,116,104,111,100,32,111,110, - 32,97,108,108,32,112,97,116,104,32,101,110,116,114,121,32, - 102,105,110,100,101,114,115,10,32,32,32,32,32,32,32,32, - 115,116,111,114,101,100,32,105,110,32,115,121,115,46,112,97, - 116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,104, - 101,115,32,40,119,104,101,114,101,32,105,109,112,108,101,109, - 101,110,116,101,100,41,46,218,17,105,110,118,97,108,105,100, - 97,116,101,95,99,97,99,104,101,115,78,41,5,114,7,0, - 0,0,218,19,112,97,116,104,95,105,109,112,111,114,116,101, - 114,95,99,97,99,104,101,218,6,118,97,108,117,101,115,114, - 115,0,0,0,114,250,0,0,0,41,2,114,170,0,0,0, - 218,6,102,105,110,100,101,114,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,250,0,0,0,12,4,0,0, - 115,6,0,0,0,0,4,22,1,15,1,122,28,80,97,116, - 104,70,105,110,100,101,114,46,105,110,118,97,108,105,100,97, - 116,101,95,99,97,99,104,101,115,99,2,0,0,0,0,0, - 0,0,3,0,0,0,12,0,0,0,67,0,0,0,115,107, - 0,0,0,116,0,0,106,1,0,100,1,0,107,9,0,114, - 41,0,116,0,0,106,1,0,12,114,41,0,116,2,0,106, - 3,0,100,2,0,116,4,0,131,2,0,1,120,59,0,116, - 0,0,106,1,0,68,93,44,0,125,2,0,121,14,0,124, - 2,0,124,1,0,131,1,0,83,87,113,51,0,4,116,5, - 0,107,10,0,114,94,0,1,1,1,119,51,0,89,113,51, - 0,88,113,51,0,87,100,1,0,83,100,1,0,83,41,3, - 122,113,83,101,97,114,99,104,32,115,101,113,117,101,110,99, - 101,32,111,102,32,104,111,111,107,115,32,102,111,114,32,97, - 32,102,105,110,100,101,114,32,102,111,114,32,39,112,97,116, - 104,39,46,10,10,32,32,32,32,32,32,32,32,73,102,32, - 39,104,111,111,107,115,39,32,105,115,32,102,97,108,115,101, - 32,116,104,101,110,32,117,115,101,32,115,121,115,46,112,97, - 116,104,95,104,111,111,107,115,46,10,10,32,32,32,32,32, - 32,32,32,78,122,23,115,121,115,46,112,97,116,104,95,104, - 111,111,107,115,32,105,115,32,101,109,112,116,121,41,6,114, - 7,0,0,0,218,10,112,97,116,104,95,104,111,111,107,115, - 114,60,0,0,0,114,61,0,0,0,114,125,0,0,0,114, - 107,0,0,0,41,3,114,170,0,0,0,114,35,0,0,0, - 90,4,104,111,111,107,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,11,95,112,97,116,104,95,104,111,111, - 107,115,20,4,0,0,115,16,0,0,0,0,7,25,1,16, - 1,16,1,3,1,14,1,13,1,12,2,122,22,80,97,116, - 104,70,105,110,100,101,114,46,95,112,97,116,104,95,104,111, - 111,107,115,99,2,0,0,0,0,0,0,0,3,0,0,0, - 19,0,0,0,67,0,0,0,115,123,0,0,0,124,1,0, - 100,1,0,107,2,0,114,53,0,121,16,0,116,0,0,106, - 1,0,131,0,0,125,1,0,87,110,22,0,4,116,2,0, - 107,10,0,114,52,0,1,1,1,100,2,0,83,89,110,1, - 0,88,121,17,0,116,3,0,106,4,0,124,1,0,25,125, - 2,0,87,110,46,0,4,116,5,0,107,10,0,114,118,0, - 1,1,1,124,0,0,106,6,0,124,1,0,131,1,0,125, - 2,0,124,2,0,116,3,0,106,4,0,124,1,0,60,89, - 110,1,0,88,124,2,0,83,41,3,122,210,71,101,116,32, - 116,104,101,32,102,105,110,100,101,114,32,102,111,114,32,116, - 104,101,32,112,97,116,104,32,101,110,116,114,121,32,102,114, - 111,109,32,115,121,115,46,112,97,116,104,95,105,109,112,111, - 114,116,101,114,95,99,97,99,104,101,46,10,10,32,32,32, - 32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104, - 32,101,110,116,114,121,32,105,115,32,110,111,116,32,105,110, - 32,116,104,101,32,99,97,99,104,101,44,32,102,105,110,100, - 32,116,104,101,32,97,112,112,114,111,112,114,105,97,116,101, - 32,102,105,110,100,101,114,10,32,32,32,32,32,32,32,32, - 97,110,100,32,99,97,99,104,101,32,105,116,46,32,73,102, - 32,110,111,32,102,105,110,100,101,114,32,105,115,32,97,118, - 97,105,108,97,98,108,101,44,32,115,116,111,114,101,32,78, - 111,110,101,46,10,10,32,32,32,32,32,32,32,32,114,30, - 0,0,0,78,41,7,114,3,0,0,0,114,45,0,0,0, - 218,17,70,105,108,101,78,111,116,70,111,117,110,100,69,114, - 114,111,114,114,7,0,0,0,114,251,0,0,0,114,137,0, - 0,0,114,255,0,0,0,41,3,114,170,0,0,0,114,35, - 0,0,0,114,253,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,20,95,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,37,4,0, - 0,115,22,0,0,0,0,8,12,1,3,1,16,1,13,3, - 9,1,3,1,17,1,13,1,15,1,18,1,122,31,80,97, - 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,99,3,0, - 0,0,0,0,0,0,6,0,0,0,3,0,0,0,67,0, - 0,0,115,119,0,0,0,116,0,0,124,2,0,100,1,0, - 131,2,0,114,39,0,124,2,0,106,1,0,124,1,0,131, - 1,0,92,2,0,125,3,0,125,4,0,110,21,0,124,2, - 0,106,2,0,124,1,0,131,1,0,125,3,0,103,0,0, - 125,4,0,124,3,0,100,0,0,107,9,0,114,88,0,116, - 3,0,106,4,0,124,1,0,124,3,0,131,2,0,83,116, - 3,0,106,5,0,124,1,0,100,0,0,131,2,0,125,5, - 0,124,4,0,124,5,0,95,6,0,124,5,0,83,41,2, - 78,114,124,0,0,0,41,7,114,115,0,0,0,114,124,0, - 0,0,114,182,0,0,0,114,121,0,0,0,114,179,0,0, - 0,114,160,0,0,0,114,156,0,0,0,41,6,114,170,0, - 0,0,114,126,0,0,0,114,253,0,0,0,114,127,0,0, - 0,114,128,0,0,0,114,164,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,16,95,108,101,103, - 97,99,121,95,103,101,116,95,115,112,101,99,59,4,0,0, - 115,18,0,0,0,0,4,15,1,24,2,15,1,6,1,12, - 1,16,1,18,1,9,1,122,27,80,97,116,104,70,105,110, - 100,101,114,46,95,108,101,103,97,99,121,95,103,101,116,95, - 115,112,101,99,78,99,4,0,0,0,0,0,0,0,9,0, - 0,0,5,0,0,0,67,0,0,0,115,243,0,0,0,103, - 0,0,125,4,0,120,230,0,124,2,0,68,93,191,0,125, - 5,0,116,0,0,124,5,0,116,1,0,116,2,0,102,2, - 0,131,2,0,115,43,0,113,13,0,124,0,0,106,3,0, - 124,5,0,131,1,0,125,6,0,124,6,0,100,1,0,107, - 9,0,114,13,0,116,4,0,124,6,0,100,2,0,131,2, - 0,114,106,0,124,6,0,106,5,0,124,1,0,124,3,0, - 131,2,0,125,7,0,110,18,0,124,0,0,106,6,0,124, - 1,0,124,6,0,131,2,0,125,7,0,124,7,0,100,1, - 0,107,8,0,114,139,0,113,13,0,124,7,0,106,7,0, - 100,1,0,107,9,0,114,158,0,124,7,0,83,124,7,0, - 106,8,0,125,8,0,124,8,0,100,1,0,107,8,0,114, - 191,0,116,9,0,100,3,0,131,1,0,130,1,0,124,4, - 0,106,10,0,124,8,0,131,1,0,1,113,13,0,87,116, - 11,0,106,12,0,124,1,0,100,1,0,131,2,0,125,7, - 0,124,4,0,124,7,0,95,8,0,124,7,0,83,100,1, - 0,83,41,4,122,63,70,105,110,100,32,116,104,101,32,108, - 111,97,100,101,114,32,111,114,32,110,97,109,101,115,112,97, - 99,101,95,112,97,116,104,32,102,111,114,32,116,104,105,115, - 32,109,111,100,117,108,101,47,112,97,99,107,97,103,101,32, - 110,97,109,101,46,78,114,181,0,0,0,122,19,115,112,101, - 99,32,109,105,115,115,105,110,103,32,108,111,97,100,101,114, - 41,13,114,143,0,0,0,114,69,0,0,0,218,5,98,121, - 116,101,115,114,1,1,0,0,114,115,0,0,0,114,181,0, - 0,0,114,2,1,0,0,114,127,0,0,0,114,156,0,0, - 0,114,107,0,0,0,114,149,0,0,0,114,121,0,0,0, - 114,160,0,0,0,41,9,114,170,0,0,0,114,126,0,0, - 0,114,35,0,0,0,114,180,0,0,0,218,14,110,97,109, - 101,115,112,97,99,101,95,112,97,116,104,90,5,101,110,116, - 114,121,114,253,0,0,0,114,164,0,0,0,114,128,0,0, + 67,0,0,0,115,7,0,0,0,124,0,0,106,0,0,83, + 41,1,122,58,82,101,116,117,114,110,32,116,104,101,32,112, + 97,116,104,32,116,111,32,116,104,101,32,115,111,117,114,99, + 101,32,102,105,108,101,32,97,115,32,102,111,117,110,100,32, + 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, + 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,9,95,103,101,116,95,115,112,101,99,74,4,0,0,115, - 40,0,0,0,0,5,6,1,13,1,21,1,3,1,15,1, - 12,1,15,1,21,2,18,1,12,1,3,1,15,1,4,1, - 9,1,12,1,12,5,17,2,18,1,9,1,122,20,80,97, - 116,104,70,105,110,100,101,114,46,95,103,101,116,95,115,112, - 101,99,99,4,0,0,0,0,0,0,0,6,0,0,0,4, - 0,0,0,67,0,0,0,115,140,0,0,0,124,2,0,100, - 1,0,107,8,0,114,21,0,116,0,0,106,1,0,125,2, - 0,124,0,0,106,2,0,124,1,0,124,2,0,124,3,0, - 131,3,0,125,4,0,124,4,0,100,1,0,107,8,0,114, - 58,0,100,1,0,83,124,4,0,106,3,0,100,1,0,107, - 8,0,114,132,0,124,4,0,106,4,0,125,5,0,124,5, - 0,114,125,0,100,2,0,124,4,0,95,5,0,116,6,0, - 124,1,0,124,5,0,124,0,0,106,2,0,131,3,0,124, - 4,0,95,4,0,124,4,0,83,100,1,0,83,110,4,0, - 124,4,0,83,100,1,0,83,41,3,122,98,102,105,110,100, - 32,116,104,101,32,109,111,100,117,108,101,32,111,110,32,115, - 121,115,46,112,97,116,104,32,111,114,32,39,112,97,116,104, - 39,32,98,97,115,101,100,32,111,110,32,115,121,115,46,112, - 97,116,104,95,104,111,111,107,115,32,97,110,100,10,32,32, - 32,32,32,32,32,32,115,121,115,46,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,46,78,90, - 9,110,97,109,101,115,112,97,99,101,41,7,114,7,0,0, - 0,114,35,0,0,0,114,5,1,0,0,114,127,0,0,0, - 114,156,0,0,0,114,158,0,0,0,114,230,0,0,0,41, - 6,114,170,0,0,0,114,126,0,0,0,114,35,0,0,0, - 114,180,0,0,0,114,164,0,0,0,114,4,1,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,181, - 0,0,0,106,4,0,0,115,26,0,0,0,0,4,12,1, - 9,1,21,1,12,1,4,1,15,1,9,1,6,3,9,1, - 24,1,4,2,7,2,122,20,80,97,116,104,70,105,110,100, - 101,114,46,102,105,110,100,95,115,112,101,99,99,3,0,0, - 0,0,0,0,0,4,0,0,0,3,0,0,0,67,0,0, - 0,115,41,0,0,0,124,0,0,106,0,0,124,1,0,124, - 2,0,131,2,0,125,3,0,124,3,0,100,1,0,107,8, - 0,114,34,0,100,1,0,83,124,3,0,106,1,0,83,41, - 2,122,170,102,105,110,100,32,116,104,101,32,109,111,100,117, - 108,101,32,111,110,32,115,121,115,46,112,97,116,104,32,111, - 114,32,39,112,97,116,104,39,32,98,97,115,101,100,32,111, - 110,32,115,121,115,46,112,97,116,104,95,104,111,111,107,115, - 32,97,110,100,10,32,32,32,32,32,32,32,32,115,121,115, - 46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99, - 97,99,104,101,46,10,10,32,32,32,32,32,32,32,32,84, - 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, - 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, - 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,2, - 114,181,0,0,0,114,127,0,0,0,41,4,114,170,0,0, - 0,114,126,0,0,0,114,35,0,0,0,114,164,0,0,0, + 114,151,0,0,0,154,3,0,0,115,2,0,0,0,0,3, + 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, + 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, + 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, + 106,0,0,0,114,107,0,0,0,114,179,0,0,0,114,207, + 0,0,0,114,209,0,0,0,114,180,0,0,0,114,185,0, + 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, + 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 182,0,0,0,128,4,0,0,115,8,0,0,0,0,8,18, - 1,12,1,4,1,122,22,80,97,116,104,70,105,110,100,101, - 114,46,102,105,110,100,95,109,111,100,117,108,101,41,12,114, - 112,0,0,0,114,111,0,0,0,114,113,0,0,0,114,114, - 0,0,0,114,183,0,0,0,114,250,0,0,0,114,255,0, - 0,0,114,1,1,0,0,114,2,1,0,0,114,5,1,0, - 0,114,181,0,0,0,114,182,0,0,0,114,4,0,0,0, + 218,0,0,0,107,3,0,0,115,20,0,0,0,12,6,6, + 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, + 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, + 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, + 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, + 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, + 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, + 132,0,0,90,6,0,100,8,0,100,9,0,132,0,0,90, + 7,0,100,10,0,100,11,0,132,0,0,90,8,0,100,12, + 0,100,13,0,132,0,0,90,9,0,100,14,0,100,15,0, + 132,0,0,90,10,0,100,16,0,100,17,0,132,0,0,90, + 11,0,100,18,0,100,19,0,132,0,0,90,12,0,100,20, + 0,83,41,21,218,14,95,78,97,109,101,115,112,97,99,101, + 80,97,116,104,97,38,1,0,0,82,101,112,114,101,115,101, + 110,116,115,32,97,32,110,97,109,101,115,112,97,99,101,32, + 112,97,99,107,97,103,101,39,115,32,112,97,116,104,46,32, + 32,73,116,32,117,115,101,115,32,116,104,101,32,109,111,100, + 117,108,101,32,110,97,109,101,10,32,32,32,32,116,111,32, + 102,105,110,100,32,105,116,115,32,112,97,114,101,110,116,32, + 109,111,100,117,108,101,44,32,97,110,100,32,102,114,111,109, + 32,116,104,101,114,101,32,105,116,32,108,111,111,107,115,32, + 117,112,32,116,104,101,32,112,97,114,101,110,116,39,115,10, + 32,32,32,32,95,95,112,97,116,104,95,95,46,32,32,87, + 104,101,110,32,116,104,105,115,32,99,104,97,110,103,101,115, + 44,32,116,104,101,32,109,111,100,117,108,101,39,115,32,111, + 119,110,32,112,97,116,104,32,105,115,32,114,101,99,111,109, + 112,117,116,101,100,44,10,32,32,32,32,117,115,105,110,103, + 32,112,97,116,104,95,102,105,110,100,101,114,46,32,32,70, + 111,114,32,116,111,112,45,108,101,118,101,108,32,109,111,100, + 117,108,101,115,44,32,116,104,101,32,112,97,114,101,110,116, + 32,109,111,100,117,108,101,39,115,32,112,97,116,104,10,32, + 32,32,32,105,115,32,115,121,115,46,112,97,116,104,46,99, + 4,0,0,0,0,0,0,0,4,0,0,0,2,0,0,0, + 67,0,0,0,115,52,0,0,0,124,1,0,124,0,0,95, + 0,0,124,2,0,124,0,0,95,1,0,116,2,0,124,0, + 0,106,3,0,131,0,0,131,1,0,124,0,0,95,4,0, + 124,3,0,124,0,0,95,5,0,100,0,0,83,41,1,78, + 41,6,218,5,95,110,97,109,101,218,5,95,112,97,116,104, + 114,93,0,0,0,218,16,95,103,101,116,95,112,97,114,101, + 110,116,95,112,97,116,104,218,17,95,108,97,115,116,95,112, + 97,114,101,110,116,95,112,97,116,104,218,12,95,112,97,116, + 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, + 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, + 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,179,0,0,0,167,3,0,0,115,8, + 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, + 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, + 0,3,0,0,0,67,0,0,0,115,53,0,0,0,124,0, + 0,106,0,0,106,1,0,100,1,0,131,1,0,92,3,0, + 125,1,0,125,2,0,125,3,0,124,2,0,100,2,0,107, + 2,0,114,43,0,100,6,0,83,124,1,0,100,5,0,102, + 2,0,83,41,7,122,62,82,101,116,117,114,110,115,32,97, + 32,116,117,112,108,101,32,111,102,32,40,112,97,114,101,110, + 116,45,109,111,100,117,108,101,45,110,97,109,101,44,32,112, + 97,114,101,110,116,45,112,97,116,104,45,97,116,116,114,45, + 110,97,109,101,41,114,58,0,0,0,114,30,0,0,0,114, + 7,0,0,0,114,35,0,0,0,90,8,95,95,112,97,116, + 104,95,95,41,2,122,3,115,121,115,122,4,112,97,116,104, + 41,2,114,225,0,0,0,114,32,0,0,0,41,4,114,100, + 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, + 97,116,104,95,110,97,109,101,115,173,3,0,0,115,8,0, + 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, + 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, + 101,115,99,1,0,0,0,0,0,0,0,3,0,0,0,3, + 0,0,0,67,0,0,0,115,38,0,0,0,124,0,0,106, + 0,0,131,0,0,92,2,0,125,1,0,125,2,0,116,1, + 0,116,2,0,106,3,0,124,1,0,25,124,2,0,131,2, + 0,83,41,1,78,41,4,114,232,0,0,0,114,110,0,0, + 0,114,7,0,0,0,218,7,109,111,100,117,108,101,115,41, + 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, + 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, + 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,227,0,0,0,183,3, + 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, + 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, + 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, + 0,115,118,0,0,0,116,0,0,124,0,0,106,1,0,131, + 0,0,131,1,0,125,1,0,124,1,0,124,0,0,106,2, + 0,107,3,0,114,111,0,124,0,0,106,3,0,124,0,0, + 106,4,0,124,1,0,131,2,0,125,2,0,124,2,0,100, + 0,0,107,9,0,114,102,0,124,2,0,106,5,0,100,0, + 0,107,8,0,114,102,0,124,2,0,106,6,0,114,102,0, + 124,2,0,106,6,0,124,0,0,95,7,0,124,1,0,124, + 0,0,95,2,0,124,0,0,106,7,0,83,41,1,78,41, + 8,114,93,0,0,0,114,227,0,0,0,114,228,0,0,0, + 114,229,0,0,0,114,225,0,0,0,114,120,0,0,0,114, + 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, + 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,187, + 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, + 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, + 117,108,97,116,101,99,1,0,0,0,0,0,0,0,1,0, + 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, + 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,200, + 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, + 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, + 2,0,0,0,67,0,0,0,115,16,0,0,0,116,0,0, + 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, + 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,7,95,95,108,101,110,95,95,203,3,0,0,115, + 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, + 0,0,0,115,16,0,0,0,100,1,0,106,0,0,124,0, + 0,106,1,0,131,1,0,83,41,2,78,122,20,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, + 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, + 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,8,95,95,114,101,112,114,95,95,206,3,0, + 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, + 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, + 0,0,67,0,0,0,115,16,0,0,0,124,1,0,124,0, + 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, + 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, + 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,209, + 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, + 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, + 2,0,0,0,2,0,0,0,67,0,0,0,115,20,0,0, + 0,124,0,0,106,0,0,106,1,0,124,1,0,131,1,0, + 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, + 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 249,0,0,0,8,4,0,0,115,22,0,0,0,12,2,6, - 2,18,8,18,17,18,22,18,15,3,1,18,31,3,1,21, - 21,3,1,114,249,0,0,0,99,0,0,0,0,0,0,0, - 0,0,0,0,0,3,0,0,0,64,0,0,0,115,133,0, - 0,0,101,0,0,90,1,0,100,0,0,90,2,0,100,1, - 0,90,3,0,100,2,0,100,3,0,132,0,0,90,4,0, - 100,4,0,100,5,0,132,0,0,90,5,0,101,6,0,90, - 7,0,100,6,0,100,7,0,132,0,0,90,8,0,100,8, - 0,100,9,0,132,0,0,90,9,0,100,10,0,100,11,0, - 100,12,0,132,1,0,90,10,0,100,13,0,100,14,0,132, - 0,0,90,11,0,101,12,0,100,15,0,100,16,0,132,0, - 0,131,1,0,90,13,0,100,17,0,100,18,0,132,0,0, - 90,14,0,100,10,0,83,41,19,218,10,70,105,108,101,70, - 105,110,100,101,114,122,172,70,105,108,101,45,98,97,115,101, - 100,32,102,105,110,100,101,114,46,10,10,32,32,32,32,73, - 110,116,101,114,97,99,116,105,111,110,115,32,119,105,116,104, - 32,116,104,101,32,102,105,108,101,32,115,121,115,116,101,109, - 32,97,114,101,32,99,97,99,104,101,100,32,102,111,114,32, - 112,101,114,102,111,114,109,97,110,99,101,44,32,98,101,105, - 110,103,10,32,32,32,32,114,101,102,114,101,115,104,101,100, - 32,119,104,101,110,32,116,104,101,32,100,105,114,101,99,116, - 111,114,121,32,116,104,101,32,102,105,110,100,101,114,32,105, - 115,32,104,97,110,100,108,105,110,103,32,104,97,115,32,98, - 101,101,110,32,109,111,100,105,102,105,101,100,46,10,10,32, - 32,32,32,99,2,0,0,0,0,0,0,0,5,0,0,0, - 5,0,0,0,7,0,0,0,115,122,0,0,0,103,0,0, - 125,3,0,120,52,0,124,2,0,68,93,44,0,92,2,0, - 137,0,0,125,4,0,124,3,0,106,0,0,135,0,0,102, - 1,0,100,1,0,100,2,0,134,0,0,124,4,0,68,131, - 1,0,131,1,0,1,113,13,0,87,124,3,0,124,0,0, - 95,1,0,124,1,0,112,79,0,100,3,0,124,0,0,95, - 2,0,100,6,0,124,0,0,95,3,0,116,4,0,131,0, - 0,124,0,0,95,5,0,116,4,0,131,0,0,124,0,0, - 95,6,0,100,5,0,83,41,7,122,154,73,110,105,116,105, - 97,108,105,122,101,32,119,105,116,104,32,116,104,101,32,112, - 97,116,104,32,116,111,32,115,101,97,114,99,104,32,111,110, - 32,97,110,100,32,97,32,118,97,114,105,97,98,108,101,32, - 110,117,109,98,101,114,32,111,102,10,32,32,32,32,32,32, - 32,32,50,45,116,117,112,108,101,115,32,99,111,110,116,97, - 105,110,105,110,103,32,116,104,101,32,108,111,97,100,101,114, - 32,97,110,100,32,116,104,101,32,102,105,108,101,32,115,117, - 102,102,105,120,101,115,32,116,104,101,32,108,111,97,100,101, - 114,10,32,32,32,32,32,32,32,32,114,101,99,111,103,110, - 105,122,101,115,46,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,51,0,0,0,115,27,0,0,0,124, - 0,0,93,17,0,125,1,0,124,1,0,136,0,0,102,2, - 0,86,1,113,3,0,100,0,0,83,41,1,78,114,4,0, - 0,0,41,2,114,22,0,0,0,114,225,0,0,0,41,1, - 114,127,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 227,0,0,0,157,4,0,0,115,2,0,0,0,6,0,122, - 38,70,105,108,101,70,105,110,100,101,114,46,95,95,105,110, - 105,116,95,95,46,60,108,111,99,97,108,115,62,46,60,103, - 101,110,101,120,112,114,62,114,58,0,0,0,114,29,0,0, - 0,78,114,87,0,0,0,41,7,114,149,0,0,0,218,8, - 95,108,111,97,100,101,114,115,114,35,0,0,0,218,11,95, - 112,97,116,104,95,109,116,105,109,101,218,3,115,101,116,218, - 11,95,112,97,116,104,95,99,97,99,104,101,218,19,95,114, - 101,108,97,120,101,100,95,112,97,116,104,95,99,97,99,104, - 101,41,5,114,108,0,0,0,114,35,0,0,0,218,14,108, - 111,97,100,101,114,95,100,101,116,97,105,108,115,90,7,108, - 111,97,100,101,114,115,114,166,0,0,0,114,4,0,0,0, - 41,1,114,127,0,0,0,114,5,0,0,0,114,185,0,0, - 0,151,4,0,0,115,16,0,0,0,0,4,6,1,19,1, - 36,1,9,2,15,1,9,1,12,1,122,19,70,105,108,101, - 70,105,110,100,101,114,46,95,95,105,110,105,116,95,95,99, - 1,0,0,0,0,0,0,0,1,0,0,0,2,0,0,0, - 67,0,0,0,115,13,0,0,0,100,3,0,124,0,0,95, - 0,0,100,2,0,83,41,4,122,31,73,110,118,97,108,105, - 100,97,116,101,32,116,104,101,32,100,105,114,101,99,116,111, - 114,121,32,109,116,105,109,101,46,114,29,0,0,0,78,114, - 87,0,0,0,41,1,114,8,1,0,0,41,1,114,108,0, + 157,0,0,0,212,3,0,0,115,2,0,0,0,0,1,122, + 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, + 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, + 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, + 0,0,114,232,0,0,0,114,227,0,0,0,114,234,0,0, + 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, + 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, + 0,0,0,160,3,0,0,115,20,0,0,0,12,5,6,2, + 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, + 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,100,2, + 0,132,0,0,90,3,0,101,4,0,100,3,0,100,4,0, + 132,0,0,131,1,0,90,5,0,100,5,0,100,6,0,132, + 0,0,90,6,0,100,7,0,100,8,0,132,0,0,90,7, + 0,100,9,0,100,10,0,132,0,0,90,8,0,100,11,0, + 100,12,0,132,0,0,90,9,0,100,13,0,100,14,0,132, + 0,0,90,10,0,100,15,0,100,16,0,132,0,0,90,11, + 0,100,17,0,83,41,18,218,16,95,78,97,109,101,115,112, + 97,99,101,76,111,97,100,101,114,99,4,0,0,0,0,0, + 0,0,4,0,0,0,4,0,0,0,67,0,0,0,115,25, + 0,0,0,116,0,0,124,1,0,124,2,0,124,3,0,131, + 3,0,124,0,0,95,1,0,100,0,0,83,41,1,78,41, + 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, + 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,179,0,0,0,218,3,0,0,115,2,0,0,0,0,1, + 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, + 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, + 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, + 115,16,0,0,0,100,1,0,106,0,0,124,1,0,106,1, + 0,131,1,0,83,41,2,122,115,82,101,116,117,114,110,32, + 114,101,112,114,32,102,111,114,32,116,104,101,32,109,111,100, + 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, + 101,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, + 101,99,97,116,101,100,46,32,32,84,104,101,32,105,109,112, + 111,114,116,32,109,97,99,104,105,110,101,114,121,32,100,111, + 101,115,32,116,104,101,32,106,111,98,32,105,116,115,101,108, + 102,46,10,10,32,32,32,32,32,32,32,32,122,25,60,109, + 111,100,117,108,101,32,123,33,114,125,32,40,110,97,109,101, + 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, + 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, + 109,111,100,117,108,101,95,114,101,112,114,221,3,0,0,115, + 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, + 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, + 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, + 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,153,0,0,0,230,3,0,0,115, + 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, + 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, + 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, + 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,196,0,0,0,233,3, + 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, + 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, + 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,6,0,0,0,67,0,0,0,115,22,0,0,0, + 116,0,0,100,1,0,100,2,0,100,3,0,100,4,0,100, + 5,0,131,3,1,83,41,6,78,114,30,0,0,0,122,8, + 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, + 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,181,0,0,0,236,3,0,0,115,2, + 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, + 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, + 0,67,0,0,0,115,4,0,0,0,100,1,0,83,41,2, + 122,42,85,115,101,32,100,101,102,97,117,108,116,32,115,101, + 109,97,110,116,105,99,115,32,102,111,114,32,109,111,100,117, + 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, + 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, + 0,0,239,3,0,0,115,0,0,0,0,122,30,95,78,97, + 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, + 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, + 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, + 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, + 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, + 0,242,3,0,0,115,2,0,0,0,0,1,122,28,95,78, + 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, + 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, + 35,0,0,0,116,0,0,106,1,0,100,1,0,124,0,0, + 106,2,0,131,2,0,1,116,0,0,106,3,0,124,0,0, + 124,1,0,131,2,0,83,41,2,122,98,76,111,97,100,32, + 97,32,110,97,109,101,115,112,97,99,101,32,109,111,100,117, + 108,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105, + 115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, + 101,99,97,116,101,100,46,32,32,85,115,101,32,101,120,101, + 99,95,109,111,100,117,108,101,40,41,32,105,110,115,116,101, + 97,100,46,10,10,32,32,32,32,32,32,32,32,122,38,110, + 97,109,101,115,112,97,99,101,32,109,111,100,117,108,101,32, + 108,111,97,100,101,100,32,119,105,116,104,32,112,97,116,104, + 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, + 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, + 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,187,0,0,0,245,3,0,0,115, + 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, + 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, + 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,179,0,0,0,114, + 177,0,0,0,114,242,0,0,0,114,153,0,0,0,114,196, + 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, + 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, + 217,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, + 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, + 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,101,4,0,100,2,0, + 100,3,0,132,0,0,131,1,0,90,5,0,101,4,0,100, + 4,0,100,5,0,132,0,0,131,1,0,90,6,0,101,4, + 0,100,6,0,100,7,0,132,0,0,131,1,0,90,7,0, + 101,4,0,100,8,0,100,9,0,132,0,0,131,1,0,90, + 8,0,101,4,0,100,10,0,100,11,0,100,12,0,132,1, + 0,131,1,0,90,9,0,101,4,0,100,10,0,100,10,0, + 100,13,0,100,14,0,132,2,0,131,1,0,90,10,0,101, + 4,0,100,10,0,100,15,0,100,16,0,132,1,0,131,1, + 0,90,11,0,100,10,0,83,41,17,218,10,80,97,116,104, + 70,105,110,100,101,114,122,62,77,101,116,97,32,112,97,116, + 104,32,102,105,110,100,101,114,32,102,111,114,32,115,121,115, + 46,112,97,116,104,32,97,110,100,32,112,97,99,107,97,103, + 101,32,95,95,112,97,116,104,95,95,32,97,116,116,114,105, + 98,117,116,101,115,46,99,1,0,0,0,0,0,0,0,2, + 0,0,0,4,0,0,0,67,0,0,0,115,55,0,0,0, + 120,48,0,116,0,0,106,1,0,106,2,0,131,0,0,68, + 93,31,0,125,1,0,116,3,0,124,1,0,100,1,0,131, + 2,0,114,16,0,124,1,0,106,4,0,131,0,0,1,113, + 16,0,87,100,2,0,83,41,3,122,125,67,97,108,108,32, + 116,104,101,32,105,110,118,97,108,105,100,97,116,101,95,99, + 97,99,104,101,115,40,41,32,109,101,116,104,111,100,32,111, + 110,32,97,108,108,32,112,97,116,104,32,101,110,116,114,121, + 32,102,105,110,100,101,114,115,10,32,32,32,32,32,32,32, + 32,115,116,111,114,101,100,32,105,110,32,115,121,115,46,112, + 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, + 104,101,115,32,40,119,104,101,114,101,32,105,109,112,108,101, + 109,101,110,116,101,100,41,46,218,17,105,110,118,97,108,105, + 100,97,116,101,95,99,97,99,104,101,115,78,41,5,114,7, + 0,0,0,218,19,112,97,116,104,95,105,109,112,111,114,116, + 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, + 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, + 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,244,0,0,0,7,4,0, + 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, + 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, + 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, + 0,0,0,3,0,0,0,12,0,0,0,67,0,0,0,115, + 107,0,0,0,116,0,0,106,1,0,100,1,0,107,9,0, + 114,41,0,116,0,0,106,1,0,12,114,41,0,116,2,0, + 106,3,0,100,2,0,116,4,0,131,2,0,1,120,59,0, + 116,0,0,106,1,0,68,93,44,0,125,2,0,121,14,0, + 124,2,0,124,1,0,131,1,0,83,87,113,51,0,4,116, + 5,0,107,10,0,114,94,0,1,1,1,119,51,0,89,113, + 51,0,88,113,51,0,87,100,1,0,83,100,1,0,83,41, + 3,122,113,83,101,97,114,99,104,32,115,101,113,117,101,110, + 99,101,32,111,102,32,104,111,111,107,115,32,102,111,114,32, + 97,32,102,105,110,100,101,114,32,102,111,114,32,39,112,97, + 116,104,39,46,10,10,32,32,32,32,32,32,32,32,73,102, + 32,39,104,111,111,107,115,39,32,105,115,32,102,97,108,115, + 101,32,116,104,101,110,32,117,115,101,32,115,121,115,46,112, + 97,116,104,95,104,111,111,107,115,46,10,10,32,32,32,32, + 32,32,32,32,78,122,23,115,121,115,46,112,97,116,104,95, + 104,111,111,107,115,32,105,115,32,101,109,112,116,121,41,6, + 114,7,0,0,0,218,10,112,97,116,104,95,104,111,111,107, + 115,114,60,0,0,0,114,61,0,0,0,114,118,0,0,0, + 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, + 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, + 111,107,115,15,4,0,0,115,16,0,0,0,0,7,25,1, + 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, + 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, + 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, + 0,19,0,0,0,67,0,0,0,115,123,0,0,0,124,1, + 0,100,1,0,107,2,0,114,53,0,121,16,0,116,0,0, + 106,1,0,131,0,0,125,1,0,87,110,22,0,4,116,2, + 0,107,10,0,114,52,0,1,1,1,100,2,0,83,89,110, + 1,0,88,121,17,0,116,3,0,106,4,0,124,1,0,25, + 125,2,0,87,110,46,0,4,116,5,0,107,10,0,114,118, + 0,1,1,1,124,0,0,106,6,0,124,1,0,131,1,0, + 125,2,0,124,2,0,116,3,0,106,4,0,124,1,0,60, + 89,110,1,0,88,124,2,0,83,41,3,122,210,71,101,116, + 32,116,104,101,32,102,105,110,100,101,114,32,102,111,114,32, + 116,104,101,32,112,97,116,104,32,101,110,116,114,121,32,102, + 114,111,109,32,115,121,115,46,112,97,116,104,95,105,109,112, + 111,114,116,101,114,95,99,97,99,104,101,46,10,10,32,32, + 32,32,32,32,32,32,73,102,32,116,104,101,32,112,97,116, + 104,32,101,110,116,114,121,32,105,115,32,110,111,116,32,105, + 110,32,116,104,101,32,99,97,99,104,101,44,32,102,105,110, + 100,32,116,104,101,32,97,112,112,114,111,112,114,105,97,116, + 101,32,102,105,110,100,101,114,10,32,32,32,32,32,32,32, + 32,97,110,100,32,99,97,99,104,101,32,105,116,46,32,73, + 102,32,110,111,32,102,105,110,100,101,114,32,105,115,32,97, + 118,97,105,108,97,98,108,101,44,32,115,116,111,114,101,32, + 78,111,110,101,46,10,10,32,32,32,32,32,32,32,32,114, + 30,0,0,0,78,41,7,114,3,0,0,0,114,45,0,0, + 0,218,17,70,105,108,101,78,111,116,70,111,117,110,100,69, + 114,114,111,114,114,7,0,0,0,114,245,0,0,0,114,131, + 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, + 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,32,4, + 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, + 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, + 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,99,3, + 0,0,0,0,0,0,0,6,0,0,0,3,0,0,0,67, + 0,0,0,115,119,0,0,0,116,0,0,124,2,0,100,1, + 0,131,2,0,114,39,0,124,2,0,106,1,0,124,1,0, + 131,1,0,92,2,0,125,3,0,125,4,0,110,21,0,124, + 2,0,106,2,0,124,1,0,131,1,0,125,3,0,103,0, + 0,125,4,0,124,3,0,100,0,0,107,9,0,114,88,0, + 116,3,0,106,4,0,124,1,0,124,3,0,131,2,0,83, + 116,3,0,106,5,0,124,1,0,100,0,0,131,2,0,125, + 5,0,124,4,0,124,5,0,95,6,0,124,5,0,83,41, + 2,78,114,117,0,0,0,41,7,114,108,0,0,0,114,117, + 0,0,0,114,176,0,0,0,114,114,0,0,0,114,173,0, + 0,0,114,154,0,0,0,114,150,0,0,0,41,6,114,164, + 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, + 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, + 103,97,99,121,95,103,101,116,95,115,112,101,99,54,4,0, + 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, + 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, + 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, + 95,115,112,101,99,78,99,4,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,243,0,0,0, + 103,0,0,125,4,0,120,230,0,124,2,0,68,93,191,0, + 125,5,0,116,0,0,124,5,0,116,1,0,116,2,0,102, + 2,0,131,2,0,115,43,0,113,13,0,124,0,0,106,3, + 0,124,5,0,131,1,0,125,6,0,124,6,0,100,1,0, + 107,9,0,114,13,0,116,4,0,124,6,0,100,2,0,131, + 2,0,114,106,0,124,6,0,106,5,0,124,1,0,124,3, + 0,131,2,0,125,7,0,110,18,0,124,0,0,106,6,0, + 124,1,0,124,6,0,131,2,0,125,7,0,124,7,0,100, + 1,0,107,8,0,114,139,0,113,13,0,124,7,0,106,7, + 0,100,1,0,107,9,0,114,158,0,124,7,0,83,124,7, + 0,106,8,0,125,8,0,124,8,0,100,1,0,107,8,0, + 114,191,0,116,9,0,100,3,0,131,1,0,130,1,0,124, + 4,0,106,10,0,124,8,0,131,1,0,1,113,13,0,87, + 116,11,0,106,12,0,124,1,0,100,1,0,131,2,0,125, + 7,0,124,4,0,124,7,0,95,8,0,124,7,0,83,100, + 1,0,83,41,4,122,63,70,105,110,100,32,116,104,101,32, + 108,111,97,100,101,114,32,111,114,32,110,97,109,101,115,112, + 97,99,101,95,112,97,116,104,32,102,111,114,32,116,104,105, + 115,32,109,111,100,117,108,101,47,112,97,99,107,97,103,101, + 32,110,97,109,101,46,78,114,175,0,0,0,122,19,115,112, + 101,99,32,109,105,115,115,105,110,103,32,108,111,97,100,101, + 114,41,13,114,137,0,0,0,114,69,0,0,0,218,5,98, + 121,116,101,115,114,251,0,0,0,114,108,0,0,0,114,175, + 0,0,0,114,252,0,0,0,114,120,0,0,0,114,150,0, + 0,0,114,99,0,0,0,114,143,0,0,0,114,114,0,0, + 0,114,154,0,0,0,41,9,114,164,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,174,0,0,0,218,14,110,97, + 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, + 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,250,0,0,0,165,4,0,0,115,2,0,0,0,0, - 2,122,28,70,105,108,101,70,105,110,100,101,114,46,105,110, - 118,97,108,105,100,97,116,101,95,99,97,99,104,101,115,99, - 2,0,0,0,0,0,0,0,3,0,0,0,2,0,0,0, - 67,0,0,0,115,59,0,0,0,124,0,0,106,0,0,124, - 1,0,131,1,0,125,2,0,124,2,0,100,1,0,107,8, - 0,114,37,0,100,1,0,103,0,0,102,2,0,83,124,2, - 0,106,1,0,124,2,0,106,2,0,112,55,0,103,0,0, - 102,2,0,83,41,2,122,197,84,114,121,32,116,111,32,102, - 105,110,100,32,97,32,108,111,97,100,101,114,32,102,111,114, - 32,116,104,101,32,115,112,101,99,105,102,105,101,100,32,109, - 111,100,117,108,101,44,32,111,114,32,116,104,101,32,110,97, - 109,101,115,112,97,99,101,10,32,32,32,32,32,32,32,32, - 112,97,99,107,97,103,101,32,112,111,114,116,105,111,110,115, - 46,32,82,101,116,117,114,110,115,32,40,108,111,97,100,101, - 114,44,32,108,105,115,116,45,111,102,45,112,111,114,116,105, - 111,110,115,41,46,10,10,32,32,32,32,32,32,32,32,84, - 104,105,115,32,109,101,116,104,111,100,32,105,115,32,100,101, - 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, - 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,3, - 114,181,0,0,0,114,127,0,0,0,114,156,0,0,0,41, - 3,114,108,0,0,0,114,126,0,0,0,114,164,0,0,0, + 0,218,9,95,103,101,116,95,115,112,101,99,69,4,0,0, + 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, + 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, + 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, + 97,116,104,70,105,110,100,101,114,46,95,103,101,116,95,115, + 112,101,99,99,4,0,0,0,0,0,0,0,6,0,0,0, + 4,0,0,0,67,0,0,0,115,140,0,0,0,124,2,0, + 100,1,0,107,8,0,114,21,0,116,0,0,106,1,0,125, + 2,0,124,0,0,106,2,0,124,1,0,124,2,0,124,3, + 0,131,3,0,125,4,0,124,4,0,100,1,0,107,8,0, + 114,58,0,100,1,0,83,124,4,0,106,3,0,100,1,0, + 107,8,0,114,132,0,124,4,0,106,4,0,125,5,0,124, + 5,0,114,125,0,100,2,0,124,4,0,95,5,0,116,6, + 0,124,1,0,124,5,0,124,0,0,106,2,0,131,3,0, + 124,4,0,95,4,0,124,4,0,83,100,1,0,83,110,4, + 0,124,4,0,83,100,1,0,83,41,3,122,98,102,105,110, + 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, + 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, + 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, + 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, + 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,78, + 90,9,110,97,109,101,115,112,97,99,101,41,7,114,7,0, + 0,0,114,35,0,0,0,114,255,0,0,0,114,120,0,0, + 0,114,150,0,0,0,114,152,0,0,0,114,224,0,0,0, + 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, + 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 124,0,0,0,171,4,0,0,115,8,0,0,0,0,7,15, - 1,12,1,10,1,122,22,70,105,108,101,70,105,110,100,101, - 114,46,102,105,110,100,95,108,111,97,100,101,114,99,6,0, - 0,0,0,0,0,0,7,0,0,0,7,0,0,0,67,0, - 0,0,115,40,0,0,0,124,1,0,124,2,0,124,3,0, - 131,2,0,125,6,0,116,0,0,124,2,0,124,3,0,100, - 1,0,124,6,0,100,2,0,124,4,0,131,2,2,83,41, - 3,78,114,127,0,0,0,114,156,0,0,0,41,1,114,167, - 0,0,0,41,7,114,108,0,0,0,114,165,0,0,0,114, - 126,0,0,0,114,35,0,0,0,90,4,115,109,115,108,114, - 180,0,0,0,114,127,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,5,1,0,0,183,4,0, - 0,115,6,0,0,0,0,1,15,1,18,1,122,20,70,105, - 108,101,70,105,110,100,101,114,46,95,103,101,116,95,115,112, - 101,99,78,99,3,0,0,0,0,0,0,0,14,0,0,0, - 15,0,0,0,67,0,0,0,115,234,1,0,0,100,1,0, - 125,3,0,124,1,0,106,0,0,100,2,0,131,1,0,100, - 3,0,25,125,4,0,121,34,0,116,1,0,124,0,0,106, - 2,0,112,49,0,116,3,0,106,4,0,131,0,0,131,1, - 0,106,5,0,125,5,0,87,110,24,0,4,116,6,0,107, - 10,0,114,85,0,1,1,1,100,10,0,125,5,0,89,110, - 1,0,88,124,5,0,124,0,0,106,7,0,107,3,0,114, - 120,0,124,0,0,106,8,0,131,0,0,1,124,5,0,124, - 0,0,95,7,0,116,9,0,131,0,0,114,153,0,124,0, - 0,106,10,0,125,6,0,124,4,0,106,11,0,131,0,0, - 125,7,0,110,15,0,124,0,0,106,12,0,125,6,0,124, - 4,0,125,7,0,124,7,0,124,6,0,107,6,0,114,45, - 1,116,13,0,124,0,0,106,2,0,124,4,0,131,2,0, - 125,8,0,120,100,0,124,0,0,106,14,0,68,93,77,0, - 92,2,0,125,9,0,125,10,0,100,5,0,124,9,0,23, - 125,11,0,116,13,0,124,8,0,124,11,0,131,2,0,125, - 12,0,116,15,0,124,12,0,131,1,0,114,208,0,124,0, - 0,106,16,0,124,10,0,124,1,0,124,12,0,124,8,0, - 103,1,0,124,2,0,131,5,0,83,113,208,0,87,116,17, - 0,124,8,0,131,1,0,125,3,0,120,123,0,124,0,0, - 106,14,0,68,93,112,0,92,2,0,125,9,0,125,10,0, - 116,13,0,124,0,0,106,2,0,124,4,0,124,9,0,23, - 131,2,0,125,12,0,116,18,0,100,6,0,106,19,0,124, - 12,0,131,1,0,100,7,0,100,3,0,131,1,1,1,124, - 7,0,124,9,0,23,124,6,0,107,6,0,114,55,1,116, - 15,0,124,12,0,131,1,0,114,55,1,124,0,0,106,16, - 0,124,10,0,124,1,0,124,12,0,100,8,0,124,2,0, - 131,5,0,83,113,55,1,87,124,3,0,114,230,1,116,18, - 0,100,9,0,106,19,0,124,8,0,131,1,0,131,1,0, - 1,116,20,0,106,21,0,124,1,0,100,8,0,131,2,0, - 125,13,0,124,8,0,103,1,0,124,13,0,95,22,0,124, - 13,0,83,100,8,0,83,41,11,122,125,84,114,121,32,116, - 111,32,102,105,110,100,32,97,32,108,111,97,100,101,114,32, - 102,111,114,32,116,104,101,32,115,112,101,99,105,102,105,101, - 100,32,109,111,100,117,108,101,44,32,111,114,32,116,104,101, - 32,110,97,109,101,115,112,97,99,101,10,32,32,32,32,32, - 32,32,32,112,97,99,107,97,103,101,32,112,111,114,116,105, - 111,110,115,46,32,82,101,116,117,114,110,115,32,40,108,111, - 97,100,101,114,44,32,108,105,115,116,45,111,102,45,112,111, - 114,116,105,111,110,115,41,46,70,114,58,0,0,0,114,56, - 0,0,0,114,29,0,0,0,114,185,0,0,0,122,9,116, - 114,121,105,110,103,32,123,125,114,98,0,0,0,78,122,25, - 112,111,115,115,105,98,108,101,32,110,97,109,101,115,112,97, - 99,101,32,102,111,114,32,123,125,114,87,0,0,0,41,23, - 114,32,0,0,0,114,39,0,0,0,114,35,0,0,0,114, - 3,0,0,0,114,45,0,0,0,114,219,0,0,0,114,40, - 0,0,0,114,8,1,0,0,218,11,95,102,105,108,108,95, - 99,97,99,104,101,114,6,0,0,0,114,11,1,0,0,114, - 88,0,0,0,114,10,1,0,0,114,28,0,0,0,114,7, - 1,0,0,114,44,0,0,0,114,5,1,0,0,114,46,0, - 0,0,114,105,0,0,0,114,47,0,0,0,114,121,0,0, - 0,114,160,0,0,0,114,156,0,0,0,41,14,114,108,0, - 0,0,114,126,0,0,0,114,180,0,0,0,90,12,105,115, - 95,110,97,109,101,115,112,97,99,101,90,11,116,97,105,108, - 95,109,111,100,117,108,101,114,133,0,0,0,90,5,99,97, - 99,104,101,90,12,99,97,99,104,101,95,109,111,100,117,108, - 101,90,9,98,97,115,101,95,112,97,116,104,114,225,0,0, - 0,114,165,0,0,0,90,13,105,110,105,116,95,102,105,108, - 101,110,97,109,101,90,9,102,117,108,108,95,112,97,116,104, - 114,164,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,181,0,0,0,188,4,0,0,115,68,0, - 0,0,0,3,6,1,19,1,3,1,34,1,13,1,11,1, - 15,1,10,1,9,2,9,1,9,1,15,2,9,1,6,2, - 12,1,18,1,22,1,10,1,15,1,12,1,32,4,12,2, - 22,1,22,1,25,1,16,1,12,1,29,1,6,1,19,1, - 18,1,12,1,4,1,122,20,70,105,108,101,70,105,110,100, - 101,114,46,102,105,110,100,95,115,112,101,99,99,1,0,0, - 0,0,0,0,0,9,0,0,0,13,0,0,0,67,0,0, - 0,115,11,1,0,0,124,0,0,106,0,0,125,1,0,121, - 31,0,116,1,0,106,2,0,124,1,0,112,33,0,116,1, - 0,106,3,0,131,0,0,131,1,0,125,2,0,87,110,33, - 0,4,116,4,0,116,5,0,116,6,0,102,3,0,107,10, - 0,114,75,0,1,1,1,103,0,0,125,2,0,89,110,1, - 0,88,116,7,0,106,8,0,106,9,0,100,1,0,131,1, - 0,115,112,0,116,10,0,124,2,0,131,1,0,124,0,0, - 95,11,0,110,111,0,116,10,0,131,0,0,125,3,0,120, - 90,0,124,2,0,68,93,82,0,125,4,0,124,4,0,106, - 12,0,100,2,0,131,1,0,92,3,0,125,5,0,125,6, - 0,125,7,0,124,6,0,114,191,0,100,3,0,106,13,0, - 124,5,0,124,7,0,106,14,0,131,0,0,131,2,0,125, - 8,0,110,6,0,124,5,0,125,8,0,124,3,0,106,15, - 0,124,8,0,131,1,0,1,113,128,0,87,124,3,0,124, - 0,0,95,11,0,116,7,0,106,8,0,106,9,0,116,16, - 0,131,1,0,114,7,1,100,4,0,100,5,0,132,0,0, - 124,2,0,68,131,1,0,124,0,0,95,17,0,100,6,0, - 83,41,7,122,68,70,105,108,108,32,116,104,101,32,99,97, - 99,104,101,32,111,102,32,112,111,116,101,110,116,105,97,108, - 32,109,111,100,117,108,101,115,32,97,110,100,32,112,97,99, - 107,97,103,101,115,32,102,111,114,32,116,104,105,115,32,100, - 105,114,101,99,116,111,114,121,46,114,0,0,0,0,114,58, - 0,0,0,122,5,123,125,46,123,125,99,1,0,0,0,0, - 0,0,0,2,0,0,0,3,0,0,0,83,0,0,0,115, - 28,0,0,0,104,0,0,124,0,0,93,18,0,125,1,0, - 124,1,0,106,0,0,131,0,0,146,2,0,113,6,0,83, - 114,4,0,0,0,41,1,114,88,0,0,0,41,2,114,22, - 0,0,0,90,2,102,110,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,250,9,60,115,101,116,99,111,109,112, - 62,6,5,0,0,115,2,0,0,0,9,0,122,41,70,105, - 108,101,70,105,110,100,101,114,46,95,102,105,108,108,95,99, - 97,99,104,101,46,60,108,111,99,97,108,115,62,46,60,115, - 101,116,99,111,109,112,62,78,41,18,114,35,0,0,0,114, - 3,0,0,0,90,7,108,105,115,116,100,105,114,114,45,0, - 0,0,114,0,1,0,0,218,15,80,101,114,109,105,115,115, - 105,111,110,69,114,114,111,114,218,18,78,111,116,65,68,105, - 114,101,99,116,111,114,121,69,114,114,111,114,114,7,0,0, - 0,114,8,0,0,0,114,9,0,0,0,114,9,1,0,0, - 114,10,1,0,0,114,83,0,0,0,114,47,0,0,0,114, - 88,0,0,0,218,3,97,100,100,114,10,0,0,0,114,11, - 1,0,0,41,9,114,108,0,0,0,114,35,0,0,0,90, - 8,99,111,110,116,101,110,116,115,90,21,108,111,119,101,114, - 95,115,117,102,102,105,120,95,99,111,110,116,101,110,116,115, - 114,245,0,0,0,114,106,0,0,0,114,237,0,0,0,114, - 225,0,0,0,90,8,110,101,119,95,110,97,109,101,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,13,1, - 0,0,233,4,0,0,115,34,0,0,0,0,2,9,1,3, - 1,31,1,22,3,11,3,18,1,18,7,9,1,13,1,24, - 1,6,1,27,2,6,1,17,1,9,1,18,1,122,22,70, - 105,108,101,70,105,110,100,101,114,46,95,102,105,108,108,95, - 99,97,99,104,101,99,1,0,0,0,0,0,0,0,3,0, - 0,0,3,0,0,0,7,0,0,0,115,25,0,0,0,135, - 0,0,135,1,0,102,2,0,100,1,0,100,2,0,134,0, - 0,125,2,0,124,2,0,83,41,3,97,20,1,0,0,65, - 32,99,108,97,115,115,32,109,101,116,104,111,100,32,119,104, - 105,99,104,32,114,101,116,117,114,110,115,32,97,32,99,108, - 111,115,117,114,101,32,116,111,32,117,115,101,32,111,110,32, - 115,121,115,46,112,97,116,104,95,104,111,111,107,10,32,32, - 32,32,32,32,32,32,119,104,105,99,104,32,119,105,108,108, - 32,114,101,116,117,114,110,32,97,110,32,105,110,115,116,97, - 110,99,101,32,117,115,105,110,103,32,116,104,101,32,115,112, - 101,99,105,102,105,101,100,32,108,111,97,100,101,114,115,32, - 97,110,100,32,116,104,101,32,112,97,116,104,10,32,32,32, - 32,32,32,32,32,99,97,108,108,101,100,32,111,110,32,116, - 104,101,32,99,108,111,115,117,114,101,46,10,10,32,32,32, - 32,32,32,32,32,73,102,32,116,104,101,32,112,97,116,104, - 32,99,97,108,108,101,100,32,111,110,32,116,104,101,32,99, - 108,111,115,117,114,101,32,105,115,32,110,111,116,32,97,32, - 100,105,114,101,99,116,111,114,121,44,32,73,109,112,111,114, - 116,69,114,114,111,114,32,105,115,10,32,32,32,32,32,32, - 32,32,114,97,105,115,101,100,46,10,10,32,32,32,32,32, - 32,32,32,99,1,0,0,0,0,0,0,0,1,0,0,0, - 4,0,0,0,19,0,0,0,115,43,0,0,0,116,0,0, - 124,0,0,131,1,0,115,30,0,116,1,0,100,1,0,100, - 2,0,124,0,0,131,1,1,130,1,0,136,0,0,124,0, - 0,136,1,0,140,1,0,83,41,3,122,45,80,97,116,104, - 32,104,111,111,107,32,102,111,114,32,105,109,112,111,114,116, - 108,105,98,46,109,97,99,104,105,110,101,114,121,46,70,105, - 108,101,70,105,110,100,101,114,46,122,30,111,110,108,121,32, - 100,105,114,101,99,116,111,114,105,101,115,32,97,114,101,32, - 115,117,112,112,111,114,116,101,100,114,35,0,0,0,41,2, - 114,46,0,0,0,114,107,0,0,0,41,1,114,35,0,0, - 0,41,2,114,170,0,0,0,114,12,1,0,0,114,4,0, - 0,0,114,5,0,0,0,218,24,112,97,116,104,95,104,111, - 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, - 114,18,5,0,0,115,6,0,0,0,0,2,12,1,18,1, - 122,54,70,105,108,101,70,105,110,100,101,114,46,112,97,116, - 104,95,104,111,111,107,46,60,108,111,99,97,108,115,62,46, - 112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,105, - 108,101,70,105,110,100,101,114,114,4,0,0,0,41,3,114, - 170,0,0,0,114,12,1,0,0,114,18,1,0,0,114,4, - 0,0,0,41,2,114,170,0,0,0,114,12,1,0,0,114, - 5,0,0,0,218,9,112,97,116,104,95,104,111,111,107,8, - 5,0,0,115,4,0,0,0,0,10,21,6,122,20,70,105, - 108,101,70,105,110,100,101,114,46,112,97,116,104,95,104,111, - 111,107,99,1,0,0,0,0,0,0,0,1,0,0,0,2, - 0,0,0,67,0,0,0,115,16,0,0,0,100,1,0,106, - 0,0,124,0,0,106,1,0,131,1,0,83,41,2,78,122, - 16,70,105,108,101,70,105,110,100,101,114,40,123,33,114,125, - 41,41,2,114,47,0,0,0,114,35,0,0,0,41,1,114, - 108,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,26,5,0,0,115,2,0,0, - 0,0,1,122,19,70,105,108,101,70,105,110,100,101,114,46, - 95,95,114,101,112,114,95,95,41,15,114,112,0,0,0,114, - 111,0,0,0,114,113,0,0,0,114,114,0,0,0,114,185, - 0,0,0,114,250,0,0,0,114,130,0,0,0,114,182,0, - 0,0,114,124,0,0,0,114,5,1,0,0,114,181,0,0, - 0,114,13,1,0,0,114,183,0,0,0,114,19,1,0,0, - 114,244,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,6,1,0,0,142,4, - 0,0,115,20,0,0,0,12,7,6,2,12,14,12,4,6, - 2,12,12,12,5,15,45,12,31,18,18,114,6,1,0,0, - 99,4,0,0,0,0,0,0,0,6,0,0,0,11,0,0, - 0,67,0,0,0,115,195,0,0,0,124,0,0,106,0,0, - 100,1,0,131,1,0,125,4,0,124,0,0,106,0,0,100, - 2,0,131,1,0,125,5,0,124,4,0,115,99,0,124,5, - 0,114,54,0,124,5,0,106,1,0,125,4,0,110,45,0, - 124,2,0,124,3,0,107,2,0,114,84,0,116,2,0,124, - 1,0,124,2,0,131,2,0,125,4,0,110,15,0,116,3, - 0,124,1,0,124,2,0,131,2,0,125,4,0,124,5,0, - 115,126,0,116,4,0,124,1,0,124,2,0,100,3,0,124, - 4,0,131,2,1,125,5,0,121,44,0,124,5,0,124,0, - 0,100,2,0,60,124,4,0,124,0,0,100,1,0,60,124, - 2,0,124,0,0,100,4,0,60,124,3,0,124,0,0,100, - 5,0,60,87,110,18,0,4,116,5,0,107,10,0,114,190, - 0,1,1,1,89,110,1,0,88,100,0,0,83,41,6,78, - 218,10,95,95,108,111,97,100,101,114,95,95,218,8,95,95, - 115,112,101,99,95,95,114,127,0,0,0,90,8,95,95,102, - 105,108,101,95,95,90,10,95,95,99,97,99,104,101,100,95, - 95,41,6,218,3,103,101,116,114,127,0,0,0,114,223,0, - 0,0,114,218,0,0,0,114,167,0,0,0,218,9,69,120, - 99,101,112,116,105,111,110,41,6,90,2,110,115,114,106,0, - 0,0,90,8,112,97,116,104,110,97,109,101,90,9,99,112, - 97,116,104,110,97,109,101,114,127,0,0,0,114,164,0,0, + 175,0,0,0,101,4,0,0,115,26,0,0,0,0,4,12, + 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, + 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, + 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, + 0,0,0,0,0,0,4,0,0,0,3,0,0,0,67,0, + 0,0,115,41,0,0,0,124,0,0,106,0,0,124,1,0, + 124,2,0,131,2,0,125,3,0,124,3,0,100,1,0,107, + 8,0,114,34,0,100,1,0,83,124,3,0,106,1,0,83, + 41,2,122,170,102,105,110,100,32,116,104,101,32,109,111,100, + 117,108,101,32,111,110,32,115,121,115,46,112,97,116,104,32, + 111,114,32,39,112,97,116,104,39,32,98,97,115,101,100,32, + 111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107, + 115,32,97,110,100,10,32,32,32,32,32,32,32,32,115,121, + 115,46,112,97,116,104,95,105,109,112,111,114,116,101,114,95, + 99,97,99,104,101,46,10,10,32,32,32,32,32,32,32,32, + 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, + 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, + 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, + 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, + 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,176,0,0,0,123,4,0,0,115,8,0,0,0,0,8, + 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, + 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, + 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, + 107,0,0,0,114,177,0,0,0,114,244,0,0,0,114,249, + 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, + 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,14,95,102,105,120,95,117,112,95,109,111,100,117,108,101, - 32,5,0,0,115,34,0,0,0,0,2,15,1,15,1,6, - 1,6,1,12,1,12,1,18,2,15,1,6,1,21,1,3, - 1,10,1,10,1,10,1,14,1,13,2,114,24,1,0,0, - 99,0,0,0,0,0,0,0,0,3,0,0,0,3,0,0, - 0,67,0,0,0,115,55,0,0,0,116,0,0,116,1,0, - 106,2,0,131,0,0,102,2,0,125,0,0,116,3,0,116, - 4,0,102,2,0,125,1,0,116,5,0,116,6,0,102,2, - 0,125,2,0,124,0,0,124,1,0,124,2,0,103,3,0, - 83,41,1,122,95,82,101,116,117,114,110,115,32,97,32,108, - 105,115,116,32,111,102,32,102,105,108,101,45,98,97,115,101, - 100,32,109,111,100,117,108,101,32,108,111,97,100,101,114,115, - 46,10,10,32,32,32,32,69,97,99,104,32,105,116,101,109, - 32,105,115,32,97,32,116,117,112,108,101,32,40,108,111,97, - 100,101,114,44,32,115,117,102,102,105,120,101,115,41,46,10, - 32,32,32,32,41,7,114,224,0,0,0,114,145,0,0,0, - 218,18,101,120,116,101,110,115,105,111,110,95,115,117,102,102, - 105,120,101,115,114,218,0,0,0,114,84,0,0,0,114,223, - 0,0,0,114,74,0,0,0,41,3,90,10,101,120,116,101, - 110,115,105,111,110,115,90,6,115,111,117,114,99,101,90,8, - 98,121,116,101,99,111,100,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,161,0,0,0,55,5,0,0, - 115,8,0,0,0,0,5,18,1,12,1,12,1,114,161,0, - 0,0,99,1,0,0,0,0,0,0,0,12,0,0,0,12, - 0,0,0,67,0,0,0,115,70,2,0,0,124,0,0,97, - 0,0,116,0,0,106,1,0,97,1,0,116,0,0,106,2, - 0,97,2,0,116,1,0,106,3,0,116,4,0,25,125,1, - 0,120,76,0,100,26,0,68,93,68,0,125,2,0,124,2, - 0,116,1,0,106,3,0,107,7,0,114,83,0,116,0,0, - 106,5,0,124,2,0,131,1,0,125,3,0,110,13,0,116, - 1,0,106,3,0,124,2,0,25,125,3,0,116,6,0,124, - 1,0,124,2,0,124,3,0,131,3,0,1,113,44,0,87, - 100,5,0,100,6,0,103,1,0,102,2,0,100,7,0,100, - 8,0,100,6,0,103,2,0,102,2,0,102,2,0,125,4, - 0,120,149,0,124,4,0,68,93,129,0,92,2,0,125,5, - 0,125,6,0,116,7,0,100,9,0,100,10,0,132,0,0, - 124,6,0,68,131,1,0,131,1,0,115,199,0,116,8,0, - 130,1,0,124,6,0,100,11,0,25,125,7,0,124,5,0, - 116,1,0,106,3,0,107,6,0,114,241,0,116,1,0,106, - 3,0,124,5,0,25,125,8,0,80,113,156,0,121,20,0, - 116,0,0,106,5,0,124,5,0,131,1,0,125,8,0,80, - 87,113,156,0,4,116,9,0,107,10,0,114,28,1,1,1, - 1,119,156,0,89,113,156,0,88,113,156,0,87,116,9,0, - 100,12,0,131,1,0,130,1,0,116,6,0,124,1,0,100, - 13,0,124,8,0,131,3,0,1,116,6,0,124,1,0,100, - 14,0,124,7,0,131,3,0,1,116,6,0,124,1,0,100, - 15,0,100,16,0,106,10,0,124,6,0,131,1,0,131,3, - 0,1,121,19,0,116,0,0,106,5,0,100,17,0,131,1, - 0,125,9,0,87,110,24,0,4,116,9,0,107,10,0,114, - 147,1,1,1,1,100,18,0,125,9,0,89,110,1,0,88, - 116,6,0,124,1,0,100,17,0,124,9,0,131,3,0,1, - 116,0,0,106,5,0,100,19,0,131,1,0,125,10,0,116, - 6,0,124,1,0,100,19,0,124,10,0,131,3,0,1,124, - 5,0,100,7,0,107,2,0,114,238,1,116,0,0,106,5, - 0,100,20,0,131,1,0,125,11,0,116,6,0,124,1,0, - 100,21,0,124,11,0,131,3,0,1,116,6,0,124,1,0, - 100,22,0,116,11,0,131,0,0,131,3,0,1,116,12,0, - 106,13,0,116,2,0,106,14,0,131,0,0,131,1,0,1, - 124,5,0,100,7,0,107,2,0,114,66,2,116,15,0,106, - 16,0,100,23,0,131,1,0,1,100,24,0,116,12,0,107, - 6,0,114,66,2,100,25,0,116,17,0,95,18,0,100,18, - 0,83,41,27,122,205,83,101,116,117,112,32,116,104,101,32, - 112,97,116,104,45,98,97,115,101,100,32,105,109,112,111,114, - 116,101,114,115,32,102,111,114,32,105,109,112,111,114,116,108, - 105,98,32,98,121,32,105,109,112,111,114,116,105,110,103,32, - 110,101,101,100,101,100,10,32,32,32,32,98,117,105,108,116, - 45,105,110,32,109,111,100,117,108,101,115,32,97,110,100,32, - 105,110,106,101,99,116,105,110,103,32,116,104,101,109,32,105, - 110,116,111,32,116,104,101,32,103,108,111,98,97,108,32,110, - 97,109,101,115,112,97,99,101,46,10,10,32,32,32,32,79, - 116,104,101,114,32,99,111,109,112,111,110,101,110,116,115,32, - 97,114,101,32,101,120,116,114,97,99,116,101,100,32,102,114, - 111,109,32,116,104,101,32,99,111,114,101,32,98,111,111,116, - 115,116,114,97,112,32,109,111,100,117,108,101,46,10,10,32, - 32,32,32,114,49,0,0,0,114,60,0,0,0,218,8,98, - 117,105,108,116,105,110,115,114,142,0,0,0,90,5,112,111, - 115,105,120,250,1,47,218,2,110,116,250,1,92,99,1,0, - 0,0,0,0,0,0,2,0,0,0,3,0,0,0,115,0, - 0,0,115,33,0,0,0,124,0,0,93,23,0,125,1,0, - 116,0,0,124,1,0,131,1,0,100,0,0,107,2,0,86, - 1,113,3,0,100,1,0,83,41,2,114,29,0,0,0,78, - 41,1,114,31,0,0,0,41,2,114,22,0,0,0,114,77, + 114,243,0,0,0,3,4,0,0,115,22,0,0,0,12,2, + 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, + 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, + 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, + 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, + 0,100,4,0,100,5,0,132,0,0,90,5,0,101,6,0, + 90,7,0,100,6,0,100,7,0,132,0,0,90,8,0,100, + 8,0,100,9,0,132,0,0,90,9,0,100,10,0,100,11, + 0,100,12,0,132,1,0,90,10,0,100,13,0,100,14,0, + 132,0,0,90,11,0,101,12,0,100,15,0,100,16,0,132, + 0,0,131,1,0,90,13,0,100,17,0,100,18,0,132,0, + 0,90,14,0,100,10,0,83,41,19,218,10,70,105,108,101, + 70,105,110,100,101,114,122,172,70,105,108,101,45,98,97,115, + 101,100,32,102,105,110,100,101,114,46,10,10,32,32,32,32, + 73,110,116,101,114,97,99,116,105,111,110,115,32,119,105,116, + 104,32,116,104,101,32,102,105,108,101,32,115,121,115,116,101, + 109,32,97,114,101,32,99,97,99,104,101,100,32,102,111,114, + 32,112,101,114,102,111,114,109,97,110,99,101,44,32,98,101, + 105,110,103,10,32,32,32,32,114,101,102,114,101,115,104,101, + 100,32,119,104,101,110,32,116,104,101,32,100,105,114,101,99, + 116,111,114,121,32,116,104,101,32,102,105,110,100,101,114,32, + 105,115,32,104,97,110,100,108,105,110,103,32,104,97,115,32, + 98,101,101,110,32,109,111,100,105,102,105,101,100,46,10,10, + 32,32,32,32,99,2,0,0,0,0,0,0,0,5,0,0, + 0,5,0,0,0,7,0,0,0,115,122,0,0,0,103,0, + 0,125,3,0,120,52,0,124,2,0,68,93,44,0,92,2, + 0,137,0,0,125,4,0,124,3,0,106,0,0,135,0,0, + 102,1,0,100,1,0,100,2,0,134,0,0,124,4,0,68, + 131,1,0,131,1,0,1,113,13,0,87,124,3,0,124,0, + 0,95,1,0,124,1,0,112,79,0,100,3,0,124,0,0, + 95,2,0,100,6,0,124,0,0,95,3,0,116,4,0,131, + 0,0,124,0,0,95,5,0,116,4,0,131,0,0,124,0, + 0,95,6,0,100,5,0,83,41,7,122,154,73,110,105,116, + 105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,32, + 112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,111, + 110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,101, + 32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,32, + 32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,116, + 97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,101, + 114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,115, + 117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,100, + 101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,103, + 110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,51,0,0,0,115,27,0,0,0, + 124,0,0,93,17,0,125,1,0,124,1,0,136,0,0,102, + 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, + 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, + 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,221,0,0,0,152,4,0,0,115,2,0,0,0,6,0, + 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, + 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, + 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, + 0,0,78,114,87,0,0,0,41,7,114,143,0,0,0,218, + 8,95,108,111,97,100,101,114,115,114,35,0,0,0,218,11, + 95,112,97,116,104,95,109,116,105,109,101,218,3,115,101,116, + 218,11,95,112,97,116,104,95,99,97,99,104,101,218,19,95, + 114,101,108,97,120,101,100,95,112,97,116,104,95,99,97,99, + 104,101,41,5,114,100,0,0,0,114,35,0,0,0,218,14, + 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, + 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, + 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, + 0,0,146,4,0,0,115,16,0,0,0,0,4,6,1,19, + 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, + 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,13,0,0,0,100,3,0,124,0,0, + 95,0,0,100,2,0,83,41,4,122,31,73,110,118,97,108, + 105,100,97,116,101,32,116,104,101,32,100,105,114,101,99,116, + 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, + 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,227,0,0,0,91,5,0,0,115,2,0,0,0, - 6,0,122,25,95,115,101,116,117,112,46,60,108,111,99,97, - 108,115,62,46,60,103,101,110,101,120,112,114,62,114,59,0, - 0,0,122,30,105,109,112,111,114,116,108,105,98,32,114,101, - 113,117,105,114,101,115,32,112,111,115,105,120,32,111,114,32, - 110,116,114,3,0,0,0,114,25,0,0,0,114,21,0,0, - 0,114,30,0,0,0,90,7,95,116,104,114,101,97,100,78, - 90,8,95,119,101,97,107,114,101,102,90,6,119,105,110,114, - 101,103,114,169,0,0,0,114,6,0,0,0,122,4,46,112, - 121,119,122,6,95,100,46,112,121,100,84,41,4,122,3,95, - 105,111,122,9,95,119,97,114,110,105,110,103,115,122,8,98, - 117,105,108,116,105,110,115,122,7,109,97,114,115,104,97,108, - 41,19,114,121,0,0,0,114,7,0,0,0,114,145,0,0, - 0,114,239,0,0,0,114,112,0,0,0,90,18,95,98,117, - 105,108,116,105,110,95,102,114,111,109,95,110,97,109,101,114, - 116,0,0,0,218,3,97,108,108,218,14,65,115,115,101,114, - 116,105,111,110,69,114,114,111,114,114,107,0,0,0,114,26, - 0,0,0,114,11,0,0,0,114,229,0,0,0,114,149,0, - 0,0,114,25,1,0,0,114,84,0,0,0,114,163,0,0, - 0,114,168,0,0,0,114,173,0,0,0,41,12,218,17,95, - 98,111,111,116,115,116,114,97,112,95,109,111,100,117,108,101, - 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, - 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, - 108,116,105,110,95,109,111,100,117,108,101,90,10,111,115,95, - 100,101,116,97,105,108,115,90,10,98,117,105,108,116,105,110, - 95,111,115,114,21,0,0,0,114,25,0,0,0,90,9,111, - 115,95,109,111,100,117,108,101,90,13,116,104,114,101,97,100, - 95,109,111,100,117,108,101,90,14,119,101,97,107,114,101,102, - 95,109,111,100,117,108,101,90,13,119,105,110,114,101,103,95, - 109,111,100,117,108,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,6,95,115,101,116,117,112,66,5,0, - 0,115,82,0,0,0,0,8,6,1,9,1,9,3,13,1, - 13,1,15,1,18,2,13,1,20,3,33,1,19,2,31,1, - 10,1,15,1,13,1,4,2,3,1,15,1,5,1,13,1, - 12,2,12,1,16,1,16,1,25,3,3,1,19,1,13,2, - 11,1,16,3,15,1,16,3,12,1,15,1,16,3,19,1, - 19,1,12,1,13,1,12,1,114,33,1,0,0,99,1,0, - 0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0, - 0,0,115,116,0,0,0,116,0,0,124,0,0,131,1,0, - 1,116,1,0,131,0,0,125,1,0,116,2,0,106,3,0, - 106,4,0,116,5,0,106,6,0,124,1,0,140,0,0,103, - 1,0,131,1,0,1,116,7,0,106,8,0,100,1,0,107, - 2,0,114,78,0,116,2,0,106,9,0,106,10,0,116,11, - 0,131,1,0,1,116,2,0,106,9,0,106,10,0,116,12, - 0,131,1,0,1,116,5,0,124,0,0,95,5,0,116,13, - 0,124,0,0,95,13,0,100,2,0,83,41,3,122,41,73, - 110,115,116,97,108,108,32,116,104,101,32,112,97,116,104,45, - 98,97,115,101,100,32,105,109,112,111,114,116,32,99,111,109, - 112,111,110,101,110,116,115,46,114,28,1,0,0,78,41,14, - 114,33,1,0,0,114,161,0,0,0,114,7,0,0,0,114, - 254,0,0,0,114,149,0,0,0,114,6,1,0,0,114,19, - 1,0,0,114,3,0,0,0,114,112,0,0,0,218,9,109, - 101,116,97,95,112,97,116,104,114,163,0,0,0,114,168,0, - 0,0,114,249,0,0,0,114,218,0,0,0,41,2,114,32, - 1,0,0,90,17,115,117,112,112,111,114,116,101,100,95,108, - 111,97,100,101,114,115,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,105,110,115,116,97,108,108,134, - 5,0,0,115,16,0,0,0,0,2,10,1,9,1,28,1, - 15,1,16,1,16,4,9,1,114,35,1,0,0,41,3,122, - 3,119,105,110,114,1,0,0,0,114,2,0,0,0,41,57, - 114,114,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 17,0,0,0,114,19,0,0,0,114,28,0,0,0,114,38, - 0,0,0,114,39,0,0,0,114,43,0,0,0,114,44,0, - 0,0,114,46,0,0,0,114,55,0,0,0,218,4,116,121, - 112,101,218,8,95,95,99,111,100,101,95,95,114,144,0,0, - 0,114,15,0,0,0,114,135,0,0,0,114,14,0,0,0, - 114,18,0,0,0,90,17,95,82,65,87,95,77,65,71,73, - 67,95,78,85,77,66,69,82,114,73,0,0,0,114,72,0, - 0,0,114,84,0,0,0,114,74,0,0,0,90,23,68,69, - 66,85,71,95,66,89,84,69,67,79,68,69,95,83,85,70, - 70,73,88,69,83,90,27,79,80,84,73,77,73,90,69,68, - 95,66,89,84,69,67,79,68,69,95,83,85,70,70,73,88, - 69,83,114,79,0,0,0,114,85,0,0,0,114,91,0,0, - 0,114,95,0,0,0,114,97,0,0,0,114,105,0,0,0, - 114,123,0,0,0,114,130,0,0,0,114,141,0,0,0,114, - 147,0,0,0,114,150,0,0,0,114,155,0,0,0,218,6, - 111,98,106,101,99,116,114,162,0,0,0,114,167,0,0,0, - 114,168,0,0,0,114,184,0,0,0,114,194,0,0,0,114, - 210,0,0,0,114,218,0,0,0,114,223,0,0,0,114,229, - 0,0,0,114,224,0,0,0,114,230,0,0,0,114,247,0, - 0,0,114,249,0,0,0,114,6,1,0,0,114,24,1,0, - 0,114,161,0,0,0,114,33,1,0,0,114,35,1,0,0, - 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,218,8,60,109,111,100,117,108,101,62,8,0, - 0,0,115,100,0,0,0,6,17,6,3,12,12,12,5,12, - 5,12,6,12,12,12,10,12,9,12,5,12,7,15,22,15, - 110,22,1,18,2,6,1,6,2,9,2,9,2,10,2,21, - 44,12,33,12,19,12,12,12,12,18,8,12,28,12,17,21, - 55,21,12,18,10,12,14,9,3,12,1,15,65,19,64,19, - 28,22,110,19,41,25,43,25,16,6,3,25,53,19,57,19, - 41,19,134,19,146,15,23,12,11,12,68, + 0,0,114,244,0,0,0,160,4,0,0,115,2,0,0,0, + 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, + 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, + 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, + 0,67,0,0,0,115,59,0,0,0,124,0,0,106,0,0, + 124,1,0,131,1,0,125,2,0,124,2,0,100,1,0,107, + 8,0,114,37,0,100,1,0,103,0,0,102,2,0,83,124, + 2,0,106,1,0,124,2,0,106,2,0,112,55,0,103,0, + 0,102,2,0,83,41,2,122,197,84,114,121,32,116,111,32, + 102,105,110,100,32,97,32,108,111,97,100,101,114,32,102,111, + 114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,32, + 109,111,100,117,108,101,44,32,111,114,32,116,104,101,32,110, + 97,109,101,115,112,97,99,101,10,32,32,32,32,32,32,32, + 32,112,97,99,107,97,103,101,32,112,111,114,116,105,111,110, + 115,46,32,82,101,116,117,114,110,115,32,40,108,111,97,100, + 101,114,44,32,108,105,115,116,45,111,102,45,112,111,114,116, + 105,111,110,115,41,46,10,10,32,32,32,32,32,32,32,32, + 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, + 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, + 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, + 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, + 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, + 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,117,0,0,0,166,4,0,0,115,8,0,0,0,0,7, + 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, + 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, + 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, + 0,0,0,115,40,0,0,0,124,1,0,124,2,0,124,3, + 0,131,2,0,125,6,0,116,0,0,124,2,0,124,3,0, + 100,1,0,124,6,0,100,2,0,124,4,0,131,2,2,83, + 41,3,78,114,120,0,0,0,114,150,0,0,0,41,1,114, + 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, + 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, + 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,255,0,0,0,178,4, + 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, + 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, + 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, + 0,15,0,0,0,67,0,0,0,115,228,1,0,0,100,1, + 0,125,3,0,124,1,0,106,0,0,100,2,0,131,1,0, + 100,3,0,25,125,4,0,121,34,0,116,1,0,124,0,0, + 106,2,0,112,49,0,116,3,0,106,4,0,131,0,0,131, + 1,0,106,5,0,125,5,0,87,110,24,0,4,116,6,0, + 107,10,0,114,85,0,1,1,1,100,10,0,125,5,0,89, + 110,1,0,88,124,5,0,124,0,0,106,7,0,107,3,0, + 114,120,0,124,0,0,106,8,0,131,0,0,1,124,5,0, + 124,0,0,95,7,0,116,9,0,131,0,0,114,153,0,124, + 0,0,106,10,0,125,6,0,124,4,0,106,11,0,131,0, + 0,125,7,0,110,15,0,124,0,0,106,12,0,125,6,0, + 124,4,0,125,7,0,124,7,0,124,6,0,107,6,0,114, + 45,1,116,13,0,124,0,0,106,2,0,124,4,0,131,2, + 0,125,8,0,120,100,0,124,0,0,106,14,0,68,93,77, + 0,92,2,0,125,9,0,125,10,0,100,5,0,124,9,0, + 23,125,11,0,116,13,0,124,8,0,124,11,0,131,2,0, + 125,12,0,116,15,0,124,12,0,131,1,0,114,208,0,124, + 0,0,106,16,0,124,10,0,124,1,0,124,12,0,124,8, + 0,103,1,0,124,2,0,131,5,0,83,113,208,0,87,116, + 17,0,124,8,0,131,1,0,125,3,0,120,120,0,124,0, + 0,106,14,0,68,93,109,0,92,2,0,125,9,0,125,10, + 0,116,13,0,124,0,0,106,2,0,124,4,0,124,9,0, + 23,131,2,0,125,12,0,116,18,0,106,19,0,100,6,0, + 124,12,0,100,7,0,100,3,0,131,2,1,1,124,7,0, + 124,9,0,23,124,6,0,107,6,0,114,55,1,116,15,0, + 124,12,0,131,1,0,114,55,1,124,0,0,106,16,0,124, + 10,0,124,1,0,124,12,0,100,8,0,124,2,0,131,5, + 0,83,113,55,1,87,124,3,0,114,224,1,116,18,0,106, + 19,0,100,9,0,124,8,0,131,2,0,1,116,18,0,106, + 20,0,124,1,0,100,8,0,131,2,0,125,13,0,124,8, + 0,103,1,0,124,13,0,95,21,0,124,13,0,83,100,8, + 0,83,41,11,122,125,84,114,121,32,116,111,32,102,105,110, + 100,32,97,32,108,111,97,100,101,114,32,102,111,114,32,116, + 104,101,32,115,112,101,99,105,102,105,101,100,32,109,111,100, + 117,108,101,44,32,111,114,32,116,104,101,32,110,97,109,101, + 115,112,97,99,101,10,32,32,32,32,32,32,32,32,112,97, + 99,107,97,103,101,32,112,111,114,116,105,111,110,115,46,32, + 82,101,116,117,114,110,115,32,40,108,111,97,100,101,114,44, + 32,108,105,115,116,45,111,102,45,112,111,114,116,105,111,110, + 115,41,46,70,114,58,0,0,0,114,56,0,0,0,114,29, + 0,0,0,114,179,0,0,0,122,9,116,114,121,105,110,103, + 32,123,125,90,9,118,101,114,98,111,115,105,116,121,78,122, + 25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,112, + 97,99,101,32,102,111,114,32,123,125,114,87,0,0,0,41, + 22,114,32,0,0,0,114,39,0,0,0,114,35,0,0,0, + 114,3,0,0,0,114,45,0,0,0,114,213,0,0,0,114, + 40,0,0,0,114,2,1,0,0,218,11,95,102,105,108,108, + 95,99,97,99,104,101,114,6,0,0,0,114,5,1,0,0, + 114,88,0,0,0,114,4,1,0,0,114,28,0,0,0,114, + 1,1,0,0,114,44,0,0,0,114,255,0,0,0,114,46, + 0,0,0,114,114,0,0,0,114,129,0,0,0,114,154,0, + 0,0,114,150,0,0,0,41,14,114,100,0,0,0,114,119, + 0,0,0,114,174,0,0,0,90,12,105,115,95,110,97,109, + 101,115,112,97,99,101,90,11,116,97,105,108,95,109,111,100, + 117,108,101,114,126,0,0,0,90,5,99,97,99,104,101,90, + 12,99,97,99,104,101,95,109,111,100,117,108,101,90,9,98, + 97,115,101,95,112,97,116,104,114,219,0,0,0,114,159,0, + 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, + 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,175,0,0,0,183,4,0,0,115,70,0,0,0,0,3, + 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, + 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, + 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, + 22,1,16,1,12,1,15,1,14,1,6,1,16,1,18,1, + 12,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114, + 46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0, + 0,0,0,9,0,0,0,13,0,0,0,67,0,0,0,115, + 11,1,0,0,124,0,0,106,0,0,125,1,0,121,31,0, + 116,1,0,106,2,0,124,1,0,112,33,0,116,1,0,106, + 3,0,131,0,0,131,1,0,125,2,0,87,110,33,0,4, + 116,4,0,116,5,0,116,6,0,102,3,0,107,10,0,114, + 75,0,1,1,1,103,0,0,125,2,0,89,110,1,0,88, + 116,7,0,106,8,0,106,9,0,100,1,0,131,1,0,115, + 112,0,116,10,0,124,2,0,131,1,0,124,0,0,95,11, + 0,110,111,0,116,10,0,131,0,0,125,3,0,120,90,0, + 124,2,0,68,93,82,0,125,4,0,124,4,0,106,12,0, + 100,2,0,131,1,0,92,3,0,125,5,0,125,6,0,125, + 7,0,124,6,0,114,191,0,100,3,0,106,13,0,124,5, + 0,124,7,0,106,14,0,131,0,0,131,2,0,125,8,0, + 110,6,0,124,5,0,125,8,0,124,3,0,106,15,0,124, + 8,0,131,1,0,1,113,128,0,87,124,3,0,124,0,0, + 95,11,0,116,7,0,106,8,0,106,9,0,116,16,0,131, + 1,0,114,7,1,100,4,0,100,5,0,132,0,0,124,2, + 0,68,131,1,0,124,0,0,95,17,0,100,6,0,83,41, + 7,122,68,70,105,108,108,32,116,104,101,32,99,97,99,104, + 101,32,111,102,32,112,111,116,101,110,116,105,97,108,32,109, + 111,100,117,108,101,115,32,97,110,100,32,112,97,99,107,97, + 103,101,115,32,102,111,114,32,116,104,105,115,32,100,105,114, + 101,99,116,111,114,121,46,114,0,0,0,0,114,58,0,0, + 0,122,5,123,125,46,123,125,99,1,0,0,0,0,0,0, + 0,2,0,0,0,3,0,0,0,83,0,0,0,115,28,0, + 0,0,104,0,0,124,0,0,93,18,0,125,1,0,124,1, + 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, + 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, + 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,2, + 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, + 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, + 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, + 99,111,109,112,62,78,41,18,114,35,0,0,0,114,3,0, + 0,0,90,7,108,105,115,116,100,105,114,114,45,0,0,0, + 114,250,0,0,0,218,15,80,101,114,109,105,115,115,105,111, + 110,69,114,114,111,114,218,18,78,111,116,65,68,105,114,101, + 99,116,111,114,121,69,114,114,111,114,114,7,0,0,0,114, + 8,0,0,0,114,9,0,0,0,114,3,1,0,0,114,4, + 1,0,0,114,83,0,0,0,114,47,0,0,0,114,88,0, + 0,0,218,3,97,100,100,114,10,0,0,0,114,5,1,0, + 0,41,9,114,100,0,0,0,114,35,0,0,0,90,8,99, + 111,110,116,101,110,116,115,90,21,108,111,119,101,114,95,115, + 117,102,102,105,120,95,99,111,110,116,101,110,116,115,114,239, + 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, + 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, + 229,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, + 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, + 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, + 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, + 99,104,101,99,1,0,0,0,0,0,0,0,3,0,0,0, + 3,0,0,0,7,0,0,0,115,25,0,0,0,135,0,0, + 135,1,0,102,2,0,100,1,0,100,2,0,134,0,0,125, + 2,0,124,2,0,83,41,3,97,20,1,0,0,65,32,99, + 108,97,115,115,32,109,101,116,104,111,100,32,119,104,105,99, + 104,32,114,101,116,117,114,110,115,32,97,32,99,108,111,115, + 117,114,101,32,116,111,32,117,115,101,32,111,110,32,115,121, + 115,46,112,97,116,104,95,104,111,111,107,10,32,32,32,32, + 32,32,32,32,119,104,105,99,104,32,119,105,108,108,32,114, + 101,116,117,114,110,32,97,110,32,105,110,115,116,97,110,99, + 101,32,117,115,105,110,103,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,108,111,97,100,101,114,115,32,97,110, + 100,32,116,104,101,32,112,97,116,104,10,32,32,32,32,32, + 32,32,32,99,97,108,108,101,100,32,111,110,32,116,104,101, + 32,99,108,111,115,117,114,101,46,10,10,32,32,32,32,32, + 32,32,32,73,102,32,116,104,101,32,112,97,116,104,32,99, + 97,108,108,101,100,32,111,110,32,116,104,101,32,99,108,111, + 115,117,114,101,32,105,115,32,110,111,116,32,97,32,100,105, + 114,101,99,116,111,114,121,44,32,73,109,112,111,114,116,69, + 114,114,111,114,32,105,115,10,32,32,32,32,32,32,32,32, + 114,97,105,115,101,100,46,10,10,32,32,32,32,32,32,32, + 32,99,1,0,0,0,0,0,0,0,1,0,0,0,4,0, + 0,0,19,0,0,0,115,43,0,0,0,116,0,0,124,0, + 0,131,1,0,115,30,0,116,1,0,100,1,0,100,2,0, + 124,0,0,131,1,1,130,1,0,136,0,0,124,0,0,136, + 1,0,140,1,0,83,41,3,122,45,80,97,116,104,32,104, + 111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105, + 98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101, + 70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105, + 114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117, + 112,112,111,114,116,101,100,114,35,0,0,0,41,2,114,46, + 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, + 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, + 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,14, + 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, + 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, + 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, + 116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101, + 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, + 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, + 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, + 0,0,218,9,112,97,116,104,95,104,111,111,107,4,5,0, + 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, + 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, + 124,0,0,106,1,0,131,1,0,83,41,2,78,122,16,70, + 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, + 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,238,0,0,0,22,5,0,0,115,2,0,0,0,0, + 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, + 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, + 0,114,244,0,0,0,114,123,0,0,0,114,176,0,0,0, + 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, + 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,0,1,0,0,137,4,0,0, + 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, + 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, + 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, + 0,0,0,115,195,0,0,0,124,0,0,106,0,0,100,1, + 0,131,1,0,125,4,0,124,0,0,106,0,0,100,2,0, + 131,1,0,125,5,0,124,4,0,115,99,0,124,5,0,114, + 54,0,124,5,0,106,1,0,125,4,0,110,45,0,124,2, + 0,124,3,0,107,2,0,114,84,0,116,2,0,124,1,0, + 124,2,0,131,2,0,125,4,0,110,15,0,116,3,0,124, + 1,0,124,2,0,131,2,0,125,4,0,124,5,0,115,126, + 0,116,4,0,124,1,0,124,2,0,100,3,0,124,4,0, + 131,2,1,125,5,0,121,44,0,124,5,0,124,0,0,100, + 2,0,60,124,4,0,124,0,0,100,1,0,60,124,2,0, + 124,0,0,100,4,0,60,124,3,0,124,0,0,100,5,0, + 60,87,110,18,0,4,116,5,0,107,10,0,114,190,0,1, + 1,1,89,110,1,0,88,100,0,0,83,41,6,78,218,10, + 95,95,108,111,97,100,101,114,95,95,218,8,95,95,115,112, + 101,99,95,95,114,120,0,0,0,90,8,95,95,102,105,108, + 101,95,95,90,10,95,95,99,97,99,104,101,100,95,95,41, + 6,218,3,103,101,116,114,120,0,0,0,114,217,0,0,0, + 114,212,0,0,0,114,161,0,0,0,218,9,69,120,99,101, + 112,116,105,111,110,41,6,90,2,110,115,114,98,0,0,0, + 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, + 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, + 95,102,105,120,95,117,112,95,109,111,100,117,108,101,28,5, + 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, + 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, + 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, + 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67, + 0,0,0,115,55,0,0,0,116,0,0,116,1,0,106,2, + 0,131,0,0,102,2,0,125,0,0,116,3,0,116,4,0, + 102,2,0,125,1,0,116,5,0,116,6,0,102,2,0,125, + 2,0,124,0,0,124,1,0,124,2,0,103,3,0,83,41, + 1,122,95,82,101,116,117,114,110,115,32,97,32,108,105,115, + 116,32,111,102,32,102,105,108,101,45,98,97,115,101,100,32, + 109,111,100,117,108,101,32,108,111,97,100,101,114,115,46,10, + 10,32,32,32,32,69,97,99,104,32,105,116,101,109,32,105, + 115,32,97,32,116,117,112,108,101,32,40,108,111,97,100,101, + 114,44,32,115,117,102,102,105,120,101,115,41,46,10,32,32, + 32,32,41,7,114,218,0,0,0,114,139,0,0,0,218,18, + 101,120,116,101,110,115,105,111,110,95,115,117,102,102,105,120, + 101,115,114,212,0,0,0,114,84,0,0,0,114,217,0,0, + 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, + 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, + 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,155,0,0,0,51,5,0,0,115,8, + 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, + 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, + 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, + 116,0,0,106,1,0,97,1,0,116,0,0,106,2,0,97, + 2,0,116,1,0,106,3,0,116,4,0,25,125,1,0,120, + 76,0,100,26,0,68,93,68,0,125,2,0,124,2,0,116, + 1,0,106,3,0,107,7,0,114,83,0,116,0,0,106,5, + 0,124,2,0,131,1,0,125,3,0,110,13,0,116,1,0, + 106,3,0,124,2,0,25,125,3,0,116,6,0,124,1,0, + 124,2,0,124,3,0,131,3,0,1,113,44,0,87,100,5, + 0,100,6,0,103,1,0,102,2,0,100,7,0,100,8,0, + 100,6,0,103,2,0,102,2,0,102,2,0,125,4,0,120, + 149,0,124,4,0,68,93,129,0,92,2,0,125,5,0,125, + 6,0,116,7,0,100,9,0,100,10,0,132,0,0,124,6, + 0,68,131,1,0,131,1,0,115,199,0,116,8,0,130,1, + 0,124,6,0,100,11,0,25,125,7,0,124,5,0,116,1, + 0,106,3,0,107,6,0,114,241,0,116,1,0,106,3,0, + 124,5,0,25,125,8,0,80,113,156,0,121,20,0,116,0, + 0,106,5,0,124,5,0,131,1,0,125,8,0,80,87,113, + 156,0,4,116,9,0,107,10,0,114,28,1,1,1,1,119, + 156,0,89,113,156,0,88,113,156,0,87,116,9,0,100,12, + 0,131,1,0,130,1,0,116,6,0,124,1,0,100,13,0, + 124,8,0,131,3,0,1,116,6,0,124,1,0,100,14,0, + 124,7,0,131,3,0,1,116,6,0,124,1,0,100,15,0, + 100,16,0,106,10,0,124,6,0,131,1,0,131,3,0,1, + 121,19,0,116,0,0,106,5,0,100,17,0,131,1,0,125, + 9,0,87,110,24,0,4,116,9,0,107,10,0,114,147,1, + 1,1,1,100,18,0,125,9,0,89,110,1,0,88,116,6, + 0,124,1,0,100,17,0,124,9,0,131,3,0,1,116,0, + 0,106,5,0,100,19,0,131,1,0,125,10,0,116,6,0, + 124,1,0,100,19,0,124,10,0,131,3,0,1,124,5,0, + 100,7,0,107,2,0,114,238,1,116,0,0,106,5,0,100, + 20,0,131,1,0,125,11,0,116,6,0,124,1,0,100,21, + 0,124,11,0,131,3,0,1,116,6,0,124,1,0,100,22, + 0,116,11,0,131,0,0,131,3,0,1,116,12,0,106,13, + 0,116,2,0,106,14,0,131,0,0,131,1,0,1,124,5, + 0,100,7,0,107,2,0,114,66,2,116,15,0,106,16,0, + 100,23,0,131,1,0,1,100,24,0,116,12,0,107,6,0, + 114,66,2,100,25,0,116,17,0,95,18,0,100,18,0,83, + 41,27,122,205,83,101,116,117,112,32,116,104,101,32,112,97, + 116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,101, + 114,115,32,102,111,114,32,105,109,112,111,114,116,108,105,98, + 32,98,121,32,105,109,112,111,114,116,105,110,103,32,110,101, + 101,100,101,100,10,32,32,32,32,98,117,105,108,116,45,105, + 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, + 106,101,99,116,105,110,103,32,116,104,101,109,32,105,110,116, + 111,32,116,104,101,32,103,108,111,98,97,108,32,110,97,109, + 101,115,112,97,99,101,46,10,10,32,32,32,32,79,116,104, + 101,114,32,99,111,109,112,111,110,101,110,116,115,32,97,114, + 101,32,101,120,116,114,97,99,116,101,100,32,102,114,111,109, + 32,116,104,101,32,99,111,114,101,32,98,111,111,116,115,116, + 114,97,112,32,109,111,100,117,108,101,46,10,10,32,32,32, + 32,114,49,0,0,0,114,60,0,0,0,218,8,98,117,105, + 108,116,105,110,115,114,136,0,0,0,90,5,112,111,115,105, + 120,250,1,47,218,2,110,116,250,1,92,99,1,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,0, + 115,33,0,0,0,124,0,0,93,23,0,125,1,0,116,0, + 0,124,1,0,131,1,0,100,0,0,107,2,0,86,1,113, + 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, + 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,221,0,0,0,87,5,0,0,115,2,0,0,0,6,0, + 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, + 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, + 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, + 105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,116, + 114,3,0,0,0,114,25,0,0,0,114,21,0,0,0,114, + 30,0,0,0,90,7,95,116,104,114,101,97,100,78,90,8, + 95,119,101,97,107,114,101,102,90,6,119,105,110,114,101,103, + 114,163,0,0,0,114,6,0,0,0,122,4,46,112,121,119, + 122,6,95,100,46,112,121,100,84,41,4,122,3,95,105,111, + 122,9,95,119,97,114,110,105,110,103,115,122,8,98,117,105, + 108,116,105,110,115,122,7,109,97,114,115,104,97,108,41,19, + 114,114,0,0,0,114,7,0,0,0,114,139,0,0,0,114, + 233,0,0,0,114,105,0,0,0,90,18,95,98,117,105,108, + 116,105,110,95,102,114,111,109,95,110,97,109,101,114,109,0, + 0,0,218,3,97,108,108,218,14,65,115,115,101,114,116,105, + 111,110,69,114,114,111,114,114,99,0,0,0,114,26,0,0, + 0,114,11,0,0,0,114,223,0,0,0,114,143,0,0,0, + 114,19,1,0,0,114,84,0,0,0,114,157,0,0,0,114, + 162,0,0,0,114,167,0,0,0,41,12,218,17,95,98,111, + 111,116,115,116,114,97,112,95,109,111,100,117,108,101,90,11, + 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, + 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, + 105,110,95,109,111,100,117,108,101,90,10,111,115,95,100,101, + 116,97,105,108,115,90,10,98,117,105,108,116,105,110,95,111, + 115,114,21,0,0,0,114,25,0,0,0,90,9,111,115,95, + 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, + 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, + 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, + 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,6,95,115,101,116,117,112,62,5,0,0,115, + 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, + 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, + 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, + 12,1,16,1,16,1,25,3,3,1,19,1,13,2,11,1, + 16,3,15,1,16,3,12,1,15,1,16,3,19,1,19,1, + 12,1,13,1,12,1,114,27,1,0,0,99,1,0,0,0, + 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, + 115,116,0,0,0,116,0,0,124,0,0,131,1,0,1,116, + 1,0,131,0,0,125,1,0,116,2,0,106,3,0,106,4, + 0,116,5,0,106,6,0,124,1,0,140,0,0,103,1,0, + 131,1,0,1,116,7,0,106,8,0,100,1,0,107,2,0, + 114,78,0,116,2,0,106,9,0,106,10,0,116,11,0,131, + 1,0,1,116,2,0,106,9,0,106,10,0,116,12,0,131, + 1,0,1,116,5,0,124,0,0,95,5,0,116,13,0,124, + 0,0,95,13,0,100,2,0,83,41,3,122,41,73,110,115, + 116,97,108,108,32,116,104,101,32,112,97,116,104,45,98,97, + 115,101,100,32,105,109,112,111,114,116,32,99,111,109,112,111, + 110,101,110,116,115,46,114,22,1,0,0,78,41,14,114,27, + 1,0,0,114,155,0,0,0,114,7,0,0,0,114,248,0, + 0,0,114,143,0,0,0,114,0,1,0,0,114,13,1,0, + 0,114,3,0,0,0,114,105,0,0,0,218,9,109,101,116, + 97,95,112,97,116,104,114,157,0,0,0,114,162,0,0,0, + 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, + 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, + 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,8,95,105,110,115,116,97,108,108,130,5,0, + 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, + 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, + 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, + 0,0,0,114,10,0,0,0,114,11,0,0,0,114,17,0, + 0,0,114,19,0,0,0,114,28,0,0,0,114,38,0,0, + 0,114,39,0,0,0,114,43,0,0,0,114,44,0,0,0, + 114,46,0,0,0,114,55,0,0,0,218,4,116,121,112,101, + 218,8,95,95,99,111,100,101,95,95,114,138,0,0,0,114, + 15,0,0,0,114,128,0,0,0,114,14,0,0,0,114,18, + 0,0,0,90,17,95,82,65,87,95,77,65,71,73,67,95, + 78,85,77,66,69,82,114,73,0,0,0,114,72,0,0,0, + 114,84,0,0,0,114,74,0,0,0,90,23,68,69,66,85, + 71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73, + 88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66, + 89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83, + 114,79,0,0,0,114,85,0,0,0,114,91,0,0,0,114, + 95,0,0,0,114,97,0,0,0,114,116,0,0,0,114,123, + 0,0,0,114,135,0,0,0,114,141,0,0,0,114,144,0, + 0,0,114,149,0,0,0,218,6,111,98,106,101,99,116,114, + 156,0,0,0,114,161,0,0,0,114,162,0,0,0,114,178, + 0,0,0,114,188,0,0,0,114,204,0,0,0,114,212,0, + 0,0,114,217,0,0,0,114,223,0,0,0,114,218,0,0, + 0,114,224,0,0,0,114,241,0,0,0,114,243,0,0,0, + 114,0,1,0,0,114,18,1,0,0,114,155,0,0,0,114, + 27,1,0,0,114,29,1,0,0,114,4,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, + 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, + 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, + 12,9,12,5,12,7,15,22,15,110,22,1,18,2,6,1, + 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, + 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, + 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, + 6,3,25,53,19,57,19,42,19,134,19,147,15,23,12,11, + 12,68, }; -- cgit v0.12 From 98de5340d4d782b2a0595c9fa729460a2c1b4b91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 09:43:45 +0200 Subject: Issue #25220: Create Lib/test/libregrtest/ Start to split regrtest.py into smaller parts with the creation of Lib/test/libregrtest/cmdline.py: code to handle the command line, especially parsing command line arguments. This part of the code is tested by test_regrtest. --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 340 +++++++++++++++++++++++++++++++++++++++ Lib/test/regrtest.py | 332 +------------------------------------- Lib/test/test_regrtest.py | 4 +- 4 files changed, 345 insertions(+), 332 deletions(-) create mode 100644 Lib/test/libregrtest/__init__.py create mode 100644 Lib/test/libregrtest/cmdline.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py new file mode 100644 index 0000000..554aeed --- /dev/null +++ b/Lib/test/libregrtest/__init__.py @@ -0,0 +1 @@ +from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py new file mode 100644 index 0000000..9464996 --- /dev/null +++ b/Lib/test/libregrtest/cmdline.py @@ -0,0 +1,340 @@ +import argparse +import faulthandler +import os + +from test import support + +USAGE = """\ +python -m test [options] [test_name1 [test_name2 ...]] +python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] +""" + +DESCRIPTION = """\ +Run Python regression tests. + +If no arguments or options are provided, finds all files matching +the pattern "test_*" in the Lib/test subdirectory and runs +them in alphabetical order (but see -M and -u, below, for exceptions). + +For more rigorous testing, it is useful to use the following +command line: + +python -E -Wd -m test [options] [test_name1 ...] +""" + +EPILOG = """\ +Additional option details: + +-r randomizes test execution order. You can use --randseed=int to provide a +int seed value for the randomizer; this is useful for reproducing troublesome +test orders. + +-s On the first invocation of regrtest using -s, the first test file found +or the first test file given on the command line is run, and the name of +the next test is recorded in a file named pynexttest. If run from the +Python build directory, pynexttest is located in the 'build' subdirectory, +otherwise it is located in tempfile.gettempdir(). On subsequent runs, +the test in pynexttest is run, and the next test is written to pynexttest. +When the last test has been run, pynexttest is deleted. In this way it +is possible to single step through the test files. This is useful when +doing memory analysis on the Python interpreter, which process tends to +consume too many resources to run the full regression test non-stop. + +-S is used to continue running tests after an aborted run. It will +maintain the order a standard run (ie, this assumes -r is not used). +This is useful after the tests have prematurely stopped for some external +reason and you want to start running from where you left off rather +than starting from the beginning. + +-f reads the names of tests from the file given as f's argument, one +or more test names per line. Whitespace is ignored. Blank lines and +lines beginning with '#' are ignored. This is especially useful for +whittling down failures involving interactions among tests. + +-L causes the leaks(1) command to be run just before exit if it exists. +leaks(1) is available on Mac OS X and presumably on some other +FreeBSD-derived systems. + +-R runs each test several times and examines sys.gettotalrefcount() to +see if the test appears to be leaking references. The argument should +be of the form stab:run:fname where 'stab' is the number of times the +test is run to let gettotalrefcount settle down, 'run' is the number +of times further it is run and 'fname' is the name of the file the +reports are written to. These parameters all have defaults (5, 4 and +"reflog.txt" respectively), and the minimal invocation is '-R :'. + +-M runs tests that require an exorbitant amount of memory. These tests +typically try to ascertain containers keep working when containing more than +2 billion objects, which only works on 64-bit systems. There are also some +tests that try to exhaust the address space of the process, which only makes +sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, +which is a string in the form of '2.5Gb', determines howmuch memory the +tests will limit themselves to (but they may go slightly over.) The number +shouldn't be more memory than the machine has (including swap memory). You +should also keep in mind that swap memory is generally much, much slower +than RAM, and setting memlimit to all available RAM or higher will heavily +tax the machine. On the other hand, it is no use running these tests with a +limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect +to use more than memlimit memory will be skipped. The big-memory tests +generally run very, very long. + +-u is used to specify which special resource intensive tests to run, +such as those requiring large file support or network connectivity. +The argument is a comma-separated list of words indicating the +resources to test. Currently only the following are defined: + + all - Enable all special resources. + + none - Disable all special resources (this is the default). + + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) + + curses - Tests that use curses and will modify the terminal's + state and output modes. + + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2GB of disk space temporarily. + + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. + + decimal - Test the decimal module against a large suite that + verifies compliance with standards. + + cpu - Used for certain CPU-heavy tests. + + subprocess Run all tests for the subprocess module. + + urlfetch - It is okay to download files required on testing. + + gui - Run tests that require a running GUI. + +To enable all resources except one, use '-uall,-'. For +example, to run all the tests except for the gui tests, give the +option '-uall,-gui'. +""" + + +RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', + 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') + +class _ArgParser(argparse.ArgumentParser): + + def error(self, message): + super().error(message + "\nPass -h or --help for complete help.") + + +def _create_parser(): + # Set prog to prevent the uninformative "__main__.py" from displaying in + # error messages when using "python -m test ...". + parser = _ArgParser(prog='regrtest.py', + usage=USAGE, + description=DESCRIPTION, + epilog=EPILOG, + add_help=False, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # Arguments with this clause added to its help are described further in + # the epilog's "Additional option details" section. + more_details = ' See the section at bottom for more details.' + + group = parser.add_argument_group('General options') + # We add help explicitly to control what argument group it renders under. + group.add_argument('-h', '--help', action='help', + help='show this help message and exit') + group.add_argument('--timeout', metavar='TIMEOUT', type=float, + help='dump the traceback and exit if a test takes ' + 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'is negative or equals to zero') + group.add_argument('--wait', action='store_true', + help='wait for user input, e.g., allow a debugger ' + 'to be attached') + group.add_argument('--slaveargs', metavar='ARGS') + group.add_argument('-S', '--start', metavar='START', + help='the name of the test at which to start.' + + more_details) + + group = parser.add_argument_group('Verbosity') + group.add_argument('-v', '--verbose', action='count', + help='run tests in verbose mode with output to stdout') + group.add_argument('-w', '--verbose2', action='store_true', + help='re-run failed tests in verbose mode') + group.add_argument('-W', '--verbose3', action='store_true', + help='display test output on failure') + group.add_argument('-q', '--quiet', action='store_true', + help='no output unless one or more tests fail') + group.add_argument('-o', '--slow', action='store_true', dest='print_slow', + help='print the slowest 10 tests') + group.add_argument('--header', action='store_true', + help='print header with interpreter info') + + group = parser.add_argument_group('Selecting tests') + group.add_argument('-r', '--randomize', action='store_true', + help='randomize test execution order.' + more_details) + group.add_argument('--randseed', metavar='SEED', + dest='random_seed', type=int, + help='pass a random seed to reproduce a previous ' + 'random run') + group.add_argument('-f', '--fromfile', metavar='FILE', + help='read names of tests to run from a file.' + + more_details) + group.add_argument('-x', '--exclude', action='store_true', + help='arguments are tests to *exclude*') + group.add_argument('-s', '--single', action='store_true', + help='single step through a set of tests.' + + more_details) + group.add_argument('-m', '--match', metavar='PAT', + dest='match_tests', + help='match test cases and methods with glob pattern PAT') + group.add_argument('-G', '--failfast', action='store_true', + help='fail as soon as a test fails (only with -v or -W)') + group.add_argument('-u', '--use', metavar='RES1,RES2,...', + action='append', type=resources_list, + help='specify which special resource intensive tests ' + 'to run.' + more_details) + group.add_argument('-M', '--memlimit', metavar='LIMIT', + help='run very large memory-consuming tests.' + + more_details) + group.add_argument('--testdir', metavar='DIR', + type=relative_filename, + help='execute test files in the specified directory ' + '(instead of the Python stdlib test suite)') + + group = parser.add_argument_group('Special runs') + group.add_argument('-l', '--findleaks', action='store_true', + help='if GC is available detect tests that leak memory') + group.add_argument('-L', '--runleaks', action='store_true', + help='run the leaks(1) command just before exit.' + + more_details) + group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', + type=huntrleaks, + help='search for reference leaks (needs debug build, ' + 'very slow).' + more_details) + group.add_argument('-j', '--multiprocess', metavar='PROCESSES', + dest='use_mp', type=int, + help='run PROCESSES processes at once') + group.add_argument('-T', '--coverage', action='store_true', + dest='trace', + help='turn on code coverage tracing using the trace ' + 'module') + group.add_argument('-D', '--coverdir', metavar='DIR', + type=relative_filename, + help='directory where coverage files are put') + group.add_argument('-N', '--nocoverdir', + action='store_const', const=None, dest='coverdir', + help='put coverage files alongside modules') + group.add_argument('-t', '--threshold', metavar='THRESHOLD', + type=int, + help='call gc.set_threshold(THRESHOLD)') + group.add_argument('-n', '--nowindows', action='store_true', + help='suppress error message boxes on Windows') + group.add_argument('-F', '--forever', action='store_true', + help='run the specified tests in a loop, until an ' + 'error happens') + + parser.add_argument('args', nargs=argparse.REMAINDER, + help=argparse.SUPPRESS) + + return parser + + +def relative_filename(string): + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + return os.path.join(support.SAVEDCWD, string) + + +def huntrleaks(string): + args = string.split(':') + if len(args) not in (2, 3): + raise argparse.ArgumentTypeError( + 'needs 2 or 3 colon-separated arguments') + nwarmup = int(args[0]) if args[0] else 5 + ntracked = int(args[1]) if args[1] else 4 + fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' + return nwarmup, ntracked, fname + + +def resources_list(string): + u = [x.lower() for x in string.split(',')] + for r in u: + if r == 'all' or r == 'none': + continue + if r[0] == '-': + r = r[1:] + if r not in RESOURCE_NAMES: + raise argparse.ArgumentTypeError('invalid resource: ' + r) + return u + + +def _parse_args(args, **kwargs): + # Defaults + ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, + exclude=False, single=False, randomize=False, fromfile=None, + findleaks=False, use_resources=None, trace=False, coverdir='coverage', + runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, + random_seed=None, use_mp=None, verbose3=False, forever=False, + header=False, failfast=False, match_tests=None) + for k, v in kwargs.items(): + if not hasattr(ns, k): + raise TypeError('%r is an invalid keyword argument ' + 'for this function' % k) + setattr(ns, k, v) + if ns.use_resources is None: + ns.use_resources = [] + + parser = _create_parser() + parser.parse_args(args=args, namespace=ns) + + if ns.single and ns.fromfile: + parser.error("-s and -f don't go together!") + if ns.use_mp and ns.trace: + parser.error("-T and -j don't go together!") + if ns.use_mp and ns.findleaks: + parser.error("-l and -j don't go together!") + if ns.use_mp and ns.memlimit: + parser.error("-M and -j don't go together!") + if ns.failfast and not (ns.verbose or ns.verbose3): + parser.error("-G/--failfast needs either -v or -W") + + if ns.quiet: + ns.verbose = 0 + if ns.timeout is not None: + if hasattr(faulthandler, 'dump_traceback_later'): + if ns.timeout <= 0: + ns.timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later") + ns.timeout = None + if ns.use_mp is not None: + if ns.use_mp <= 0: + # Use all cores + extras for tests that like to sleep + ns.use_mp = 2 + (os.cpu_count() or 1) + if ns.use_mp == 1: + ns.use_mp = None + if ns.use: + for a in ns.use: + for r in a: + if r == 'all': + ns.use_resources[:] = RESOURCE_NAMES + continue + if r == 'none': + del ns.use_resources[:] + continue + remove = False + if r[0] == '-': + remove = True + r = r[1:] + if remove: + if r in ns.use_resources: + ns.use_resources.remove(r) + elif r not in ns.use_resources: + ns.use_resources.append(r) + if ns.random_seed is not None: + ns.randomize = True + + return ns diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index b49e66b..51e9e18 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -6,119 +6,6 @@ Script to run Python regression tests. Run this script with -h or --help for documentation. """ -USAGE = """\ -python -m test [options] [test_name1 [test_name2 ...]] -python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] -""" - -DESCRIPTION = """\ -Run Python regression tests. - -If no arguments or options are provided, finds all files matching -the pattern "test_*" in the Lib/test subdirectory and runs -them in alphabetical order (but see -M and -u, below, for exceptions). - -For more rigorous testing, it is useful to use the following -command line: - -python -E -Wd -m test [options] [test_name1 ...] -""" - -EPILOG = """\ -Additional option details: - --r randomizes test execution order. You can use --randseed=int to provide a -int seed value for the randomizer; this is useful for reproducing troublesome -test orders. - --s On the first invocation of regrtest using -s, the first test file found -or the first test file given on the command line is run, and the name of -the next test is recorded in a file named pynexttest. If run from the -Python build directory, pynexttest is located in the 'build' subdirectory, -otherwise it is located in tempfile.gettempdir(). On subsequent runs, -the test in pynexttest is run, and the next test is written to pynexttest. -When the last test has been run, pynexttest is deleted. In this way it -is possible to single step through the test files. This is useful when -doing memory analysis on the Python interpreter, which process tends to -consume too many resources to run the full regression test non-stop. - --S is used to continue running tests after an aborted run. It will -maintain the order a standard run (ie, this assumes -r is not used). -This is useful after the tests have prematurely stopped for some external -reason and you want to start running from where you left off rather -than starting from the beginning. - --f reads the names of tests from the file given as f's argument, one -or more test names per line. Whitespace is ignored. Blank lines and -lines beginning with '#' are ignored. This is especially useful for -whittling down failures involving interactions among tests. - --L causes the leaks(1) command to be run just before exit if it exists. -leaks(1) is available on Mac OS X and presumably on some other -FreeBSD-derived systems. - --R runs each test several times and examines sys.gettotalrefcount() to -see if the test appears to be leaking references. The argument should -be of the form stab:run:fname where 'stab' is the number of times the -test is run to let gettotalrefcount settle down, 'run' is the number -of times further it is run and 'fname' is the name of the file the -reports are written to. These parameters all have defaults (5, 4 and -"reflog.txt" respectively), and the minimal invocation is '-R :'. - --M runs tests that require an exorbitant amount of memory. These tests -typically try to ascertain containers keep working when containing more than -2 billion objects, which only works on 64-bit systems. There are also some -tests that try to exhaust the address space of the process, which only makes -sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit, -which is a string in the form of '2.5Gb', determines howmuch memory the -tests will limit themselves to (but they may go slightly over.) The number -shouldn't be more memory than the machine has (including swap memory). You -should also keep in mind that swap memory is generally much, much slower -than RAM, and setting memlimit to all available RAM or higher will heavily -tax the machine. On the other hand, it is no use running these tests with a -limit of less than 2.5Gb, and many require more than 20Gb. Tests that expect -to use more than memlimit memory will be skipped. The big-memory tests -generally run very, very long. - --u is used to specify which special resource intensive tests to run, -such as those requiring large file support or network connectivity. -The argument is a comma-separated list of words indicating the -resources to test. Currently only the following are defined: - - all - Enable all special resources. - - none - Disable all special resources (this is the default). - - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) - - curses - Tests that use curses and will modify the terminal's - state and output modes. - - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2GB of disk space temporarily. - - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. - - decimal - Test the decimal module against a large suite that - verifies compliance with standards. - - cpu - Used for certain CPU-heavy tests. - - subprocess Run all tests for the subprocess module. - - urlfetch - It is okay to download files required on testing. - - gui - Run tests that require a running GUI. - -To enable all resources except one, use '-uall,-'. For -example, to run all the tests except for the gui tests, give the -option '-uall,-gui'. -""" - # We import importlib *ASAP* in order to test #15386 import importlib @@ -153,6 +40,8 @@ try: except ImportError: multiprocessing = None +from test.libregrtest import _parse_args + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some @@ -198,9 +87,6 @@ CHILD_ERROR = -5 # error in a child process from test import support -RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', - 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui') - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -210,220 +96,6 @@ else: TEMPDIR = tempfile.gettempdir() TEMPDIR = os.path.abspath(TEMPDIR) -class _ArgParser(argparse.ArgumentParser): - - def error(self, message): - super().error(message + "\nPass -h or --help for complete help.") - -def _create_parser(): - # Set prog to prevent the uninformative "__main__.py" from displaying in - # error messages when using "python -m test ...". - parser = _ArgParser(prog='regrtest.py', - usage=USAGE, - description=DESCRIPTION, - epilog=EPILOG, - add_help=False, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # Arguments with this clause added to its help are described further in - # the epilog's "Additional option details" section. - more_details = ' See the section at bottom for more details.' - - group = parser.add_argument_group('General options') - # We add help explicitly to control what argument group it renders under. - group.add_argument('-h', '--help', action='help', - help='show this help message and exit') - group.add_argument('--timeout', metavar='TIMEOUT', type=float, - help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' - 'is negative or equals to zero') - group.add_argument('--wait', action='store_true', - help='wait for user input, e.g., allow a debugger ' - 'to be attached') - group.add_argument('--slaveargs', metavar='ARGS') - group.add_argument('-S', '--start', metavar='START', - help='the name of the test at which to start.' + - more_details) - - group = parser.add_argument_group('Verbosity') - group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') - group.add_argument('-w', '--verbose2', action='store_true', - help='re-run failed tests in verbose mode') - group.add_argument('-W', '--verbose3', action='store_true', - help='display test output on failure') - group.add_argument('-q', '--quiet', action='store_true', - help='no output unless one or more tests fail') - group.add_argument('-o', '--slow', action='store_true', dest='print_slow', - help='print the slowest 10 tests') - group.add_argument('--header', action='store_true', - help='print header with interpreter info') - - group = parser.add_argument_group('Selecting tests') - group.add_argument('-r', '--randomize', action='store_true', - help='randomize test execution order.' + more_details) - group.add_argument('--randseed', metavar='SEED', - dest='random_seed', type=int, - help='pass a random seed to reproduce a previous ' - 'random run') - group.add_argument('-f', '--fromfile', metavar='FILE', - help='read names of tests to run from a file.' + - more_details) - group.add_argument('-x', '--exclude', action='store_true', - help='arguments are tests to *exclude*') - group.add_argument('-s', '--single', action='store_true', - help='single step through a set of tests.' + - more_details) - group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', - help='match test cases and methods with glob pattern PAT') - group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') - group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, - help='specify which special resource intensive tests ' - 'to run.' + more_details) - group.add_argument('-M', '--memlimit', metavar='LIMIT', - help='run very large memory-consuming tests.' + - more_details) - group.add_argument('--testdir', metavar='DIR', - type=relative_filename, - help='execute test files in the specified directory ' - '(instead of the Python stdlib test suite)') - - group = parser.add_argument_group('Special runs') - group.add_argument('-l', '--findleaks', action='store_true', - help='if GC is available detect tests that leak memory') - group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + - more_details) - group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', - type=huntrleaks, - help='search for reference leaks (needs debug build, ' - 'very slow).' + more_details) - group.add_argument('-j', '--multiprocess', metavar='PROCESSES', - dest='use_mp', type=int, - help='run PROCESSES processes at once') - group.add_argument('-T', '--coverage', action='store_true', - dest='trace', - help='turn on code coverage tracing using the trace ' - 'module') - group.add_argument('-D', '--coverdir', metavar='DIR', - type=relative_filename, - help='directory where coverage files are put') - group.add_argument('-N', '--nocoverdir', - action='store_const', const=None, dest='coverdir', - help='put coverage files alongside modules') - group.add_argument('-t', '--threshold', metavar='THRESHOLD', - type=int, - help='call gc.set_threshold(THRESHOLD)') - group.add_argument('-n', '--nowindows', action='store_true', - help='suppress error message boxes on Windows') - group.add_argument('-F', '--forever', action='store_true', - help='run the specified tests in a loop, until an ' - 'error happens') - - parser.add_argument('args', nargs=argparse.REMAINDER, - help=argparse.SUPPRESS) - - return parser - -def relative_filename(string): - # CWD is replaced with a temporary dir before calling main(), so we - # join it with the saved CWD so it ends up where the user expects. - return os.path.join(support.SAVEDCWD, string) - -def huntrleaks(string): - args = string.split(':') - if len(args) not in (2, 3): - raise argparse.ArgumentTypeError( - 'needs 2 or 3 colon-separated arguments') - nwarmup = int(args[0]) if args[0] else 5 - ntracked = int(args[1]) if args[1] else 4 - fname = args[2] if len(args) > 2 and args[2] else 'reflog.txt' - return nwarmup, ntracked, fname - -def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: - if r == 'all' or r == 'none': - continue - if r[0] == '-': - r = r[1:] - if r not in RESOURCE_NAMES: - raise argparse.ArgumentTypeError('invalid resource: ' + r) - return u - -def _parse_args(args, **kwargs): - # Defaults - ns = argparse.Namespace(testdir=None, verbose=0, quiet=False, - exclude=False, single=False, randomize=False, fromfile=None, - findleaks=False, use_resources=None, trace=False, coverdir='coverage', - runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, - random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None) - for k, v in kwargs.items(): - if not hasattr(ns, k): - raise TypeError('%r is an invalid keyword argument ' - 'for this function' % k) - setattr(ns, k, v) - if ns.use_resources is None: - ns.use_resources = [] - - parser = _create_parser() - parser.parse_args(args=args, namespace=ns) - - if ns.single and ns.fromfile: - parser.error("-s and -f don't go together!") - if ns.use_mp and ns.trace: - parser.error("-T and -j don't go together!") - if ns.use_mp and ns.findleaks: - parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") - if ns.failfast and not (ns.verbose or ns.verbose3): - parser.error("-G/--failfast needs either -v or -W") - - if ns.quiet: - ns.verbose = 0 - if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") - ns.timeout = None - if ns.use_mp is not None: - if ns.use_mp <= 0: - # Use all cores + extras for tests that like to sleep - ns.use_mp = 2 + (os.cpu_count() or 1) - if ns.use_mp == 1: - ns.use_mp = None - if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = RESOURCE_NAMES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) - if ns.random_seed is not None: - ns.randomize = True - - return ns - - def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index a398a4f..cf4b84f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,7 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support +from test import regrtest, support, libregrtest class ParseArgsTestCase(unittest.TestCase): @@ -148,7 +148,7 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.use_resources, ['gui', 'network']) ns = regrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) - expected = list(regrtest.RESOURCE_NAMES) + expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') ns = regrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) -- cgit v0.12 From 7a84552c84867b548a72b20c177542e892c528f6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 01:30:51 -0700 Subject: Hoist constant expression out of an inner loop. --- Modules/_collectionsmodule.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d50bc42..2dcdf8b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -371,6 +371,7 @@ static PyObject * deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + PyObject *(*iternext)(PyObject *); int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ @@ -399,7 +400,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (deque->maxlen == 0) return consume_iterator(it); - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); @@ -422,8 +424,12 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque_trim_left(deque); } if (PyErr_Occurred()) { - Py_DECREF(it); - return NULL; + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } } Py_DECREF(it); Py_RETURN_NONE; @@ -436,6 +442,7 @@ static PyObject * deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; + PyObject *(*iternext)(PyObject *); int trim = (deque->maxlen != -1); /* Handle case where id(deque) == id(iterable) */ @@ -464,7 +471,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (deque->maxlen == 0) return consume_iterator(it); - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); @@ -487,8 +495,12 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque_trim_right(deque); } if (PyErr_Occurred()) { - Py_DECREF(it); - return NULL; + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } } Py_DECREF(it); Py_RETURN_NONE; -- cgit v0.12 From c22eee6b5917568933bbfec2be1069a4340efd84 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 02:14:50 -0700 Subject: Precomputing the number iterations allows the inner-loop to be vectorizable. --- Modules/_collectionsmodule.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 2dcdf8b..487c765 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -557,7 +557,7 @@ static void deque_clear(dequeobject *deque); static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) { - Py_ssize_t i, size; + Py_ssize_t i, m, size; PyObject *seq; PyObject *rv; @@ -598,7 +598,11 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) MARK_END(b->rightlink); deque->rightindex = -1; } - for ( ; i < n-1 && deque->rightindex != BLOCKLEN - 1 ; i++) { + m = n - 1 - i; + if (m > BLOCKLEN - 1 - deque->rightindex) + m = BLOCKLEN - 1 - deque->rightindex; + i += m; + while (m--) { deque->rightindex++; Py_INCREF(item); deque->rightblock->data[deque->rightindex] = item; -- cgit v0.12 From 3844fe5ed8da2755c1036deda8d7a8387170b72b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Sep 2015 10:38:01 +0200 Subject: Issue #25220: Move most regrtest.py code to libregrtest --- Lib/test/libregrtest/__init__.py | 1 + Lib/test/libregrtest/cmdline.py | 2 +- Lib/test/libregrtest/main.py | 547 +++++++++++++++++ Lib/test/libregrtest/refleak.py | 165 +++++ Lib/test/libregrtest/runtest.py | 271 +++++++++ Lib/test/libregrtest/save_env.py | 284 +++++++++ Lib/test/regrtest.py | 1225 +------------------------------------- Lib/test/test_regrtest.py | 95 +-- 8 files changed, 1318 insertions(+), 1272 deletions(-) create mode 100644 Lib/test/libregrtest/main.py create mode 100644 Lib/test/libregrtest/refleak.py create mode 100644 Lib/test/libregrtest/runtest.py create mode 100644 Lib/test/libregrtest/save_env.py diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 554aeed..a882ed2 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES +from test.libregrtest.main import main_in_temp_cwd diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 9464996..19c52a2 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,9 +1,9 @@ import argparse import faulthandler import os - from test import support + USAGE = """\ python -m test [options] [test_name1 [test_name2 ...]] python path/to/Lib/test/regrtest.py [options] [test_name1 [test_name2 ...]] diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py new file mode 100644 index 0000000..5d34cff --- /dev/null +++ b/Lib/test/libregrtest/main.py @@ -0,0 +1,547 @@ +import faulthandler +import json +import os +import re +import sys +import tempfile +import sysconfig +import signal +import random +import platform +import traceback +import unittest +from test.libregrtest.runtest import ( + findtests, runtest, run_test_in_subprocess, + STDTESTS, NOTTESTS, + PASSED, FAILED, ENV_CHANGED, SKIPPED, + RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) +from test.libregrtest.refleak import warm_caches +from test.libregrtest.cmdline import _parse_args +from test import support +try: + import threading +except ImportError: + threading = None + + +# Some times __path__ and __file__ are not absolute (e.g. while running from +# Lib/) and, if we change the CWD to run the tests in a temporary dir, some +# imports might fail. This affects only the modules imported before os.chdir(). +# These modules are searched first in sys.path[0] (so '' -- the CWD) and if +# they are found in the CWD their __file__ and __path__ will be relative (this +# happens before the chdir). All the modules imported after the chdir, are +# not found in the CWD, and since the other paths in sys.path[1:] are absolute +# (site.py absolutize them), the __file__ and __path__ will be absolute too. +# Therefore it is necessary to absolutize manually the __file__ and __path__ of +# the packages to prevent later imports to fail when the CWD is different. +for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + +# MacOSX (a.k.a. Darwin) has a default stack size that is too small +# for deeply recursive regular expressions. We see this as crashes in +# the Python test suite when running test_re.py and test_sre.py. The +# fix is to set the stack limit to 2048. +# This approach may also be useful for other Unixy platforms that +# suffer from small default stack limits. +if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +# When tests are run from the Python build directory, it is best practice +# to keep the test files in a subfolder. This eases the cleanup of leftover +# files using the "make distclean" command. +if sysconfig.is_python_build(): + TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') +else: + TEMPDIR = tempfile.gettempdir() +TEMPDIR = os.path.abspath(TEMPDIR) + + +def main(tests=None, **kwargs): + """Execute a test suite. + + This also parses command-line options and modifies its behavior + accordingly. + + tests -- a list of strings containing test names (optional) + testdir -- the directory in which to look for tests (optional) + + Users other than the Python test suite will certainly want to + specify testdir; if it's omitted, the directory containing the + Python test suite is searched for. + + If the tests argument is omitted, the tests listed on the + command-line will be used. If that's empty, too, then all *.py + files beginning with test_ will be used. + + The other default arguments (verbose, quiet, exclude, + single, randomize, findleaks, use_resources, trace, coverdir, + print_slow, and random_seed) allow programmers calling main() + directly to set the values that would normally be set by flags + on the command line. + """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + + support.record_original_stdout(sys.stdout) + + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + if ns.threshold is not None: + import gc + gc.set_threshold(ns.threshold) + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.wait: + input("Press any key to continue...") + + if ns.slaveargs is not None: + args, kwargs = json.loads(ns.slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + good = [] + bad = [] + skipped = [] + resource_denieds = [] + environment_changed = [] + interrupted = False + + if ns.findleaks: + try: + import gc + except ImportError: + print('No GC available, disabling findleaks.') + ns.findleaks = False + else: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + #gc.set_debug(gc.DEBUG_SAVEALL) + found_garbage = [] + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + if ns.single: + filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(filename, 'r') as fp: + next_test = fp.read().strip() + tests = [next_test] + except OSError: + pass + + if ns.fromfile: + tests = [] + with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + tests.extend(guts) + + # Strip .py extensions. + removepy(ns.args) + removepy(tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if ns.exclude: + for arg in ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + ns.args = [] + + # For a partial run, we do not need to clutter the output. + if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if ns.testdir: + alltests = findtests(ns.testdir, list(), set()) + else: + alltests = findtests(ns.testdir, stdtests, nottests) + + selected = tests or ns.args or alltests + if ns.single: + selected = selected[:1] + try: + next_single_test = alltests[alltests.index(selected[0])+1] + except IndexError: + next_single_test = None + # Remove all the selected tests that precede start if it's set. + if ns.start: + try: + del selected[:selected.index(ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % ns.start) + if ns.randomize: + if ns.random_seed is None: + ns.random_seed = random.randrange(10000000) + random.seed(ns.random_seed) + print("Using random seed", ns.random_seed) + random.shuffle(selected) + if ns.trace: + import trace, tempfile + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + test_times = [] + support.verbose = ns.verbose # Tell tests to be moderately quiet + support.use_resources = ns.use_resources + save_modules = sys.modules.keys() + + def accumulate_result(test, result): + ok, test_time = result + test_times.append((test_time, test)) + if ok == PASSED: + good.append(test) + elif ok == FAILED: + bad.append(test) + elif ok == ENV_CHANGED: + environment_changed.append(test) + elif ok == SKIPPED: + skipped.append(test) + elif ok == RESOURCE_DENIED: + skipped.append(test) + resource_denieds.append(test) + + if ns.forever: + def test_forever(tests=list(selected)): + while True: + for test in tests: + yield test + if bad: + return + tests = test_forever() + test_count = '' + test_count_width = 3 + else: + tests = iter(selected) + test_count = '/{}'.format(len(selected)) + test_count_width = len(test_count) - 1 + + if ns.use_mp: + try: + from threading import Thread + except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + output = Queue() + pending = MultiprocessTests(tests) + def work(): + # A worker thread. + try: + while True: + try: + test = next(pending) + except StopIteration: + output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_test_in_subprocess(test, ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + output.put((None, None, None, None)) + return + result = json.loads(result) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + output.put((None, None, None, None)) + raise + workers = [Thread(target=work) for i in range(ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + accumulate_result(test, result) + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, + len(bad), test)) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() + else: + for test_index, test in enumerate(tests, 1): + if not ns.quiet: + fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + test_count_width, test_index, test_count, len(bad), test)) + sys.stdout.flush() + if ns.trace: + # If we're tracing code coverage, then we don't exit with status + # if on a false return value from main. + tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', + globals=globals(), locals=vars()) + else: + try: + result = runtest(test, ns.verbose, ns.quiet, + ns.huntrleaks, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests) + accumulate_result(test, result) + except KeyboardInterrupt: + interrupted = True + break + if ns.findleaks: + gc.collect() + if gc.garbage: + print("Warning: test created", len(gc.garbage), end=' ') + print("uncollectable object(s).") + # move the uncollectable objects somewhere so we don't see + # them again + found_garbage.extend(gc.garbage) + del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) + for module in sys.modules.keys(): + if module not in save_modules and module.startswith("test."): + support.unload(module) + + if interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(selected) - set(good) - set(bad) - set(skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + if good and not ns.quiet: + if not bad and not skipped and not interrupted and len(good) > 1: + print("All", end=' ') + print(count(len(good), "test"), "OK.") + if ns.print_slow: + test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in test_times[:10]: + print("%s: %.1fs" % (test, time)) + if bad: + print(count(len(bad), "test"), "failed:") + printlist(bad) + if environment_changed: + print("{} altered the execution environment:".format( + count(len(environment_changed), "test"))) + printlist(environment_changed) + if skipped and not ns.quiet: + print(count(len(skipped), "test"), "skipped:") + printlist(skipped) + + if ns.verbose2 and bad: + print("Re-running failed tests in verbose mode") + for test in bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + ns.verbose = True + ok = runtest(test, True, ns.quiet, ns.huntrleaks, + timeout=ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + bad.remove(test) + else: + if bad: + print(count(len(bad), 'test'), "failed again:") + printlist(bad) + + if ns.single: + if next_single_test: + with open(filename, 'w') as fp: + fp.write(next_single_test + '\n') + else: + os.unlink(filename) + + if ns.trace: + r = tracer.results() + r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + + if ns.runleaks: + os.system("leaks %d" % os.getpid()) + + sys.exit(len(bad) > 0 or interrupted) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessTests(object): + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + import atexit + + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) + + +def removepy(names): + if not names: + return + for idx, name in enumerate(names): + basename, ext = os.path.splitext(name) + if ext == '.py': + names[idx] = basename + + +def count(n, word): + if n == 1: + return "%d %s" % (n, word) + else: + return "%d %ss" % (n, word) + + +def printlist(x, width=70, indent=4): + """Print the elements of iterable x to stdout. + + Optional arg width (default 70) is the maximum line length. + Optional arg indent (default 4) is the number of blanks with which to + begin each line. + """ + + from textwrap import fill + blanks = ' ' * indent + # Print the sorted list: 'x' may be a '--random' list or a set() + print(fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main_in_temp_cwd(): + """Run main() in a temporary working directory.""" + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + main() diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py new file mode 100644 index 0000000..dd0d05d --- /dev/null +++ b/Lib/test/libregrtest/refleak.py @@ -0,0 +1,165 @@ +import os +import re +import sys +import warnings +from inspect import isabstract +from test import support + + +def dash_R(the_module, test, indirect_test, huntrleaks): + """Run a test multiple times, looking for reference leaks. + + Returns: + False if the test didn't leak references; True if we detected refleaks. + """ + # This code is hackish and inelegant, but it seems to do the job. + import copyreg + import collections.abc + + if not hasattr(sys, 'gettotalrefcount'): + raise Exception("Tracking reference leaks requires a debug build " + "of Python") + + # Save current values for dash_R_cleanup() to restore. + fs = warnings.filters[:] + ps = copyreg.dispatch_table.copy() + pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() + abcs = {} + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = obj._abc_registry.copy() + + nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) + repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + + print("beginning", repcount, "repetitions", file=sys.stderr) + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) + sys.stderr.flush() + for i in range(repcount): + indirect_test() + alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + sys.stderr.write('.') + sys.stderr.flush() + if i >= nwarmup: + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + alloc_before, rc_before = alloc_after, rc_after + print(file=sys.stderr) + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr) + sys.stderr.flush() + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs): + import gc, copyreg + import _strptime, linecache + import urllib.parse, urllib.request, mimetypes, doctest + import struct, filecmp, collections.abc + from distutils.dir_util import _path_created + from weakref import WeakSet + + # Clear the warnings registry, so they can be displayed again + for mod in sys.modules.values(): + if hasattr(mod, '__warningregistry__'): + del mod.__warningregistry__ + + # Restore some original values. + warnings.filters[:] = fs + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(ps) + sys.path_importer_cache.clear() + sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) + + # clear type cache + sys._clear_type_cache() + + # Clear ABC registries, restoring previously saved ABC registries. + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + obj._abc_registry = abcs.get(obj, WeakSet()).copy() + obj._abc_cache.clear() + obj._abc_negative_cache.clear() + + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + # Clear assorted module caches. + _path_created.clear() + re.purge() + _strptime._regex_cache.clear() + urllib.parse.clear_cache() + urllib.request.urlcleanup() + linecache.clearcache() + mimetypes._default_mime_types() + filecmp._cache.clear() + struct._clearcache() + doctest.master = None + try: + import ctypes + except ImportError: + # Don't worry about resetting the cache if ctypes is not supported + pass + else: + ctypes._reset_cache() + + # Collect cyclic trash and read memory statistics immediately after. + func1 = sys.getallocatedblocks + func2 = sys.gettotalrefcount + gc.collect() + return func1(), func2() + + +def warm_caches(): + # char cache + s = bytes(range(256)) + for i in range(256): + s[i:i+1] + # unicode cache + x = [chr(i) for i in range(256)] + # int cache + x = list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py new file mode 100644 index 0000000..d8c0eb2 --- /dev/null +++ b/Lib/test/libregrtest/runtest.py @@ -0,0 +1,271 @@ +import faulthandler +import importlib +import io +import json +import os +import sys +import time +import traceback +import unittest +from test import support +from test.libregrtest.refleak import dash_R +from test.libregrtest.save_env import saved_test_environment + + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + + +def run_test_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) +STDTESTS = [ + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' +] + +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() + + +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + names = os.listdir(testdir) + tests = [] + others = set(stdtests) | nottests + for name in names: + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) + return stdtests + sorted(tests) + + +def runtest(test, verbose, quiet, + huntrleaks=False, use_resources=None, + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): + """Run a single test. + + test -- the name of the test + verbose -- if true, print more messages + quiet -- if true, don't print 'skipped' messages (probably redundant) + huntrleaks -- run multiple times to test for leaks; requires a debug + build; a triple corresponding to -R's three arguments + use_resources -- list of extra resources to use + output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds + failfast, match_tests -- See regrtest command-line flags for these. + + Returns the tuple result, test_time, where result is one of the constants: + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed + """ + + if use_resources is not None: + support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(timeout, exit=True) + try: + support.match_tests = match_tests + if failfast: + support.failfast = True + if output_on_failure: + support.verbose = True + + # Reuse the same instance to all calls to runtest(). Some + # tests keep a reference to sys.stdout or sys.stderr + # (eg. test_argparse). + if runtest.stringio is None: + stream = io.StringIO() + runtest.stringio = stream + else: + stream = runtest.stringio + stream.seek(0) + stream.truncate() + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + try: + sys.stdout = stream + sys.stderr = stream + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=False) + if result[0] == FAILED: + output = stream.getvalue() + orig_stderr.write(output) + orig_stderr.flush() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + else: + support.verbose = verbose # Tell tests to be moderately quiet + result = runtest_inner(test, verbose, quiet, huntrleaks, + display_failure=not verbose) + return result + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + cleanup_test_droppings(test, verbose) +runtest.stringio = None + + +def runtest_inner(test, verbose, quiet, + huntrleaks=False, display_failure=True): + support.unload(test) + + test_time = 0.0 + refleak = False # True if the test leaked references. + try: + if test.startswith('test.'): + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test + with saved_test_environment(test, verbose, quiet) as environment: + start_time = time.time() + the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + def test_runner(): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(the_module) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + support.run_unittest(tests) + test_runner() + if huntrleaks: + refleak = dash_R(the_module, test, test_runner, huntrleaks) + test_time = time.time() - start_time + except support.ResourceDenied as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return RESOURCE_DENIED, test_time + except unittest.SkipTest as msg: + if not quiet: + print(test, "skipped --", msg) + sys.stdout.flush() + return SKIPPED, test_time + except KeyboardInterrupt: + raise + except support.TestFailed as msg: + if display_failure: + print("test", test, "failed --", msg, file=sys.stderr) + else: + print("test", test, "failed", file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + except: + msg = traceback.format_exc() + print("test", test, "crashed --", msg, file=sys.stderr) + sys.stderr.flush() + return FAILED, test_time + else: + if refleak: + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time + + +def cleanup_test_droppings(testname, verbose): + import shutil + import stat + import gc + + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. + gc.collect() + + # Try to clean up junk commonly left behind. While tests shouldn't leave + # any files or directories behind, when a test fails that can be tedious + # for it to arrange. The consequences can be especially nasty on Windows, + # since if a test leaves a file open, it cannot be deleted by name (while + # there's nothing we can do about that here either, we can display the + # name of the offending test, which is a real help). + for name in (support.TESTFN, + "db_home", + ): + if not os.path.exists(name): + continue + + if os.path.isdir(name): + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise SystemError("os.path says %r exists but is neither " + "directory nor file" % name) + + if verbose: + print("%r left behind %s %r" % (testname, kind, name)) + try: + # if we have chmod, fix possible permissions problems + # that might prevent cleanup + if (hasattr(os, 'chmod')): + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as msg: + print(("%r left behind %s %r and it couldn't be " + "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py new file mode 100644 index 0000000..4f6a1aa --- /dev/null +++ b/Lib/test/libregrtest/save_env.py @@ -0,0 +1,284 @@ +import builtins +import locale +import logging +import os +import shutil +import sys +import sysconfig +import warnings +from test import support +try: + import threading +except ImportError: + threading = None +try: + import _multiprocessing, multiprocessing.process +except ImportError: + multiprocessing = None + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): + return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ + + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] + + def get_asyncore_socket_map(self): + asyncore = sys.modules.get('asyncore') + # XXX Making a copy keeps objects alive until __exit__ gets called. + return asyncore and asyncore.socket_map.copy() or {} + def restore_asyncore_socket_map(self, saved_map): + asyncore = sys.modules.get('asyncore') + if asyncore is not None: + asyncore.close_all(ignore_all=True) + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): + # Can't easily revert the logging state + pass + + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): + # Can't easily revert the logging state + pass + + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] + + # Controlling dangling references to Thread objects can make it easier + # to track reference leaks. + def get_threading__dangling(self): + if not threading: + return None + # This copies the weakrefs without making any strong reference + return threading._dangling.copy() + def restore_threading__dangling(self, saved): + if not threading: + return + threading._dangling.clear() + threading._dangling.update(saved) + + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) + + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): + return sorted(fn + ('/' if os.path.isdir(fn) else '') + for fn in os.listdir()) + def restore_files(self, saved_value): + fn = support.TESTFN + if fn not in saved_value and (fn + '/') not in saved_value: + if os.path.isfile(fn): + support.unlink(fn) + elif os.path.isdir(fn): + support.rmtree(fn) + + _lc = [getattr(locale, lc) for lc in dir(locale) + if lc.startswith('LC_')] + def get_locale(self): + pairings = [] + for lc in self._lc: + try: + pairings.append((lc, locale.setlocale(lc, None))) + except (TypeError, ValueError): + continue + return pairings + def restore_locale(self, saved): + for lc, setting in saved: + locale.setlocale(lc, setting) + + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn + + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) + + def __enter__(self): + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) + # Check for changes to the resource's value + if current != original: + self.changed = True + restore(original) + if not self.quiet: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) + return False diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 51e9e18..f01cad4 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -9,1232 +9,9 @@ Run this script with -h or --help for documentation. # We import importlib *ASAP* in order to test #15386 import importlib -import argparse -import builtins -import faulthandler -import io -import json -import locale -import logging import os -import platform -import random -import re -import shutil -import signal import sys -import sysconfig -import tempfile -import time -import traceback -import unittest -import warnings -from inspect import isabstract - -try: - import threading -except ImportError: - threading = None -try: - import _multiprocessing, multiprocessing.process -except ImportError: - multiprocessing = None - -from test.libregrtest import _parse_args - - -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - -# Test result constants. -PASSED = 1 -FAILED = 0 -ENV_CHANGED = -1 -SKIPPED = -2 -RESOURCE_DENIED = -3 -INTERRUPTED = -4 -CHILD_ERROR = -5 # error in a child process - -from test import support - -# When tests are run from the Python build directory, it is best practice -# to keep the test files in a subfolder. This eases the cleanup of leftover -# files using the "make distclean" command. -if sysconfig.is_python_build(): - TEMPDIR = os.path.join(sysconfig.get_config_var('srcdir'), 'build') -else: - TEMPDIR = tempfile.gettempdir() -TEMPDIR = os.path.abspath(TEMPDIR) - -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - -def main(tests=None, **kwargs): - """Execute a test suite. - - This also parses command-line options and modifies its behavior - accordingly. - - tests -- a list of strings containing test names (optional) - testdir -- the directory in which to look for tests (optional) - - Users other than the Python test suite will certainly want to - specify testdir; if it's omitted, the directory containing the - Python test suite is searched for. - - If the tests argument is omitted, the tests listed on the - command-line will be used. If that's empty, too, then all *.py - files beginning with test_ will be used. - - The other default arguments (verbose, quiet, exclude, - single, randomize, findleaks, use_resources, trace, coverdir, - print_slow, and random_seed) allow programmers calling main() - directly to set the values that would normally be set by flags - on the command line. - """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - - support.record_original_stdout(sys.stdout) - - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass - - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): - ok, test_time = result - test_times.append((test_time, test)) - if ok == PASSED: - good.append(test) - elif ok == FAILED: - bad.append(test) - elif ok == ENV_CHANGED: - environment_changed.append(test) - elif ok == SKIPPED: - skipped.append(test) - elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 - - if ns.use_mp: - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(tests) - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - workers = [Thread(target=work) for i in range(ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: - # If we're tracing code coverage, then we don't exit with status - # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) - else: - try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) - except KeyboardInterrupt: - interrupted = True - break - if ns.findleaks: - gc.collect() - if gc.garbage: - print("Warning: test created", len(gc.garbage), end=' ') - print("uncollectable object(s).") - # move the uncollectable objects somewhere so we don't see - # them again - found_garbage.extend(gc.garbage) - del gc.garbage[:] - # Unload the newly imported modules (best effort finalization) - for module in sys.modules.keys(): - if module not in save_modules and module.startswith("test."): - support.unload(module) - - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) - - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) - - if ns.runleaks: - os.system("leaks %d" % os.getpid()) - - sys.exit(len(bad) > 0 or interrupted) - - -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): - """Return a list of all applicable test modules.""" - testdir = findtestdir(testdir) - names = os.listdir(testdir) - tests = [] - others = set(stdtests) | nottests - for name in names: - mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and ext in (".py", "") and mod not in others: - tests.append(mod) - return stdtests + sorted(tests) - -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - -def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): - """Run a single test. - - test -- the name of the test - verbose -- if true, print more messages - quiet -- if true, don't print 'skipped' messages (probably redundant) - huntrleaks -- run multiple times to test for leaks; requires a debug - build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use - output_on_failure -- if true, display test output on failure - timeout -- dump the traceback and exit if a test takes more than - timeout seconds - failfast, match_tests -- See regrtest command-line flags for these. - - Returns the tuple result, test_time, where result is one of the constants: - INTERRUPTED KeyboardInterrupt when run under -j - RESOURCE_DENIED test skipped because resource denied - SKIPPED test skipped for some other reason - ENV_CHANGED test failed because it changed the execution environment - FAILED test failed - PASSED test passed - """ - - if use_resources is not None: - support.use_resources = use_resources - use_timeout = (timeout is not None) - if use_timeout: - faulthandler.dump_traceback_later(timeout, exit=True) - try: - support.match_tests = match_tests - if failfast: - support.failfast = True - if output_on_failure: - support.verbose = True - - # Reuse the same instance to all calls to runtest(). Some - # tests keep a reference to sys.stdout or sys.stderr - # (eg. test_argparse). - if runtest.stringio is None: - stream = io.StringIO() - runtest.stringio = stream - else: - stream = runtest.stringio - stream.seek(0) - stream.truncate() - - orig_stdout = sys.stdout - orig_stderr = sys.stderr - try: - sys.stdout = stream - sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=False) - if result[0] == FAILED: - output = stream.getvalue() - orig_stderr.write(output) - orig_stderr.flush() - finally: - sys.stdout = orig_stdout - sys.stderr = orig_stderr - else: - support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, - display_failure=not verbose) - return result - finally: - if use_timeout: - faulthandler.cancel_dump_traceback_later() - cleanup_test_droppings(test, verbose) -runtest.stringio = None - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): - return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ - - def get_warnings_filters(self): - return id(warnings.filters), warnings.filters, warnings.filters[:] - def restore_warnings_filters(self, saved_filters): - warnings.filters = saved_filters[1] - warnings.filters[:] = saved_filters[2] - - def get_asyncore_socket_map(self): - asyncore = sys.modules.get('asyncore') - # XXX Making a copy keeps objects alive until __exit__ gets called. - return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): - asyncore = sys.modules.get('asyncore') - if asyncore is not None: - asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): - # Can't easily revert the logging state - pass - - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): - # Can't easily revert the logging state - pass - - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] - - # Controlling dangling references to Thread objects can make it easier - # to track reference leaks. - def get_threading__dangling(self): - if not threading: - return None - # This copies the weakrefs without making any strong reference - return threading._dangling.copy() - def restore_threading__dangling(self, saved): - if not threading: - return - threading._dangling.clear() - threading._dangling.update(saved) - - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - - def get_sysconfig__CONFIG_VARS(self): - # make sure the dict is initialized - sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): - return sorted(fn + ('/' if os.path.isdir(fn) else '') - for fn in os.listdir()) - def restore_files(self, saved_value): - fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: - if os.path.isfile(fn): - support.unlink(fn) - elif os.path.isdir(fn): - support.rmtree(fn) - - _lc = [getattr(locale, lc) for lc in dir(locale) - if lc.startswith('LC_')] - def get_locale(self): - pairings = [] - for lc in self._lc: - try: - pairings.append((lc, locale.setlocale(lc, None))) - except (TypeError, ValueError): - continue - return pairings - def restore_locale(self, saved): - for lc, setting in saved: - locale.setlocale(lc, setting) - - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) - - def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) - # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) - return False - - -def runtest_inner(test, verbose, quiet, - huntrleaks=False, display_failure=True): - support.unload(test) - - test_time = 0.0 - refleak = False # True if the test leaked references. - try: - if test.startswith('test.'): - abstest = test - else: - # Always import it from the test package - abstest = 'test.' + test - with saved_test_environment(test, verbose, quiet) as environment: - start_time = time.time() - the_module = importlib.import_module(abstest) - # If the test has a test_main, that will run the appropriate - # tests. If not, use normal unittest test loading. - test_runner = getattr(the_module, "test_main", None) - if test_runner is None: - def test_runner(): - loader = unittest.TestLoader() - tests = loader.loadTestsFromModule(the_module) - for error in loader.errors: - print(error, file=sys.stderr) - if loader.errors: - raise Exception("errors while loading tests") - support.run_unittest(tests) - test_runner() - if huntrleaks: - refleak = dash_R(the_module, test, test_runner, huntrleaks) - test_time = time.time() - start_time - except support.ResourceDenied as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return RESOURCE_DENIED, test_time - except unittest.SkipTest as msg: - if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() - return SKIPPED, test_time - except KeyboardInterrupt: - raise - except support.TestFailed as msg: - if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) - else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - except: - msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() - return FAILED, test_time - else: - if refleak: - return FAILED, test_time - if environment.changed: - return ENV_CHANGED, test_time - return PASSED, test_time - -def cleanup_test_droppings(testname, verbose): - import shutil - import stat - import gc - - # First kill any dangling references to open files etc. - # This can also issue some ResourceWarnings which would otherwise get - # triggered during the following test run, and possibly produce failures. - gc.collect() - - # Try to clean up junk commonly left behind. While tests shouldn't leave - # any files or directories behind, when a test fails that can be tedious - # for it to arrange. The consequences can be especially nasty on Windows, - # since if a test leaves a file open, it cannot be deleted by name (while - # there's nothing we can do about that here either, we can display the - # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): - if not os.path.exists(name): - continue - - if os.path.isdir(name): - kind, nuker = "directory", shutil.rmtree - elif os.path.isfile(name): - kind, nuker = "file", os.unlink - else: - raise SystemError("os.path says %r exists but is neither " - "directory nor file" % name) - - if verbose: - print("%r left behind %s %r" % (testname, kind, name)) - try: - # if we have chmod, fix possible permissions problems - # that might prevent cleanup - if (hasattr(os, 'chmod')): - os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - nuker(name) - except Exception as msg: - print(("%r left behind %s %r and it couldn't be " - "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) - -def dash_R(the_module, test, indirect_test, huntrleaks): - """Run a test multiple times, looking for reference leaks. - - Returns: - False if the test didn't leak references; True if we detected refleaks. - """ - # This code is hackish and inelegant, but it seems to do the job. - import copyreg - import collections.abc - - if not hasattr(sys, 'gettotalrefcount'): - raise Exception("Tracking reference leaks requires a debug build " - "of Python") - - # Save current values for dash_R_cleanup() to restore. - fs = warnings.filters[:] - ps = copyreg.dispatch_table.copy() - pic = sys.path_importer_cache.copy() - try: - import zipimport - except ImportError: - zdc = None # Run unmodified on platforms without zipimport support - else: - zdc = zipimport._zip_directory_cache.copy() - abcs = {} - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - abcs[obj] = obj._abc_registry.copy() - - nwarmup, ntracked, fname = huntrleaks - fname = os.path.join(support.SAVEDCWD, fname) - repcount = nwarmup + ntracked - rc_deltas = [0] * repcount - alloc_deltas = [0] * repcount - - print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() - for i in range(repcount): - indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() - if i >= nwarmup: - rc_deltas[i] = rc_after - rc_before - alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after - print(file=sys.stderr) - # These checkers return False on success, True on failure - def check_rc_deltas(deltas): - return any(deltas) - def check_alloc_deltas(deltas): - # At least 1/3rd of 0s - if 3 * deltas.count(0) < len(deltas): - return True - # Nothing else than 1s, 0s and -1s - if not set(deltas) <= {1,0,-1}: - return True - return False - failed = False - for deltas, item_name, checker in [ - (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: - if checker(deltas): - msg = '%s leaked %s %s, sum=%s' % ( - test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() - with open(fname, "a") as refrep: - print(msg, file=refrep) - refrep.flush() - failed = True - return failed - -def dash_R_cleanup(fs, ps, pic, zdc, abcs): - import gc, copyreg - import _strptime, linecache - import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, collections.abc - from distutils.dir_util import _path_created - from weakref import WeakSet - - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - - # Restore some original values. - warnings.filters[:] = fs - copyreg.dispatch_table.clear() - copyreg.dispatch_table.update(ps) - sys.path_importer_cache.clear() - sys.path_importer_cache.update(pic) - try: - import zipimport - except ImportError: - pass # Run unmodified on platforms without zipimport support - else: - zipimport._zip_directory_cache.clear() - zipimport._zip_directory_cache.update(zdc) - - # clear type cache - sys._clear_type_cache() - - # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: - if not isabstract(abc): - continue - for obj in abc.__subclasses__() + [abc]: - obj._abc_registry = abcs.get(obj, WeakSet()).copy() - obj._abc_cache.clear() - obj._abc_negative_cache.clear() - - # Flush standard output, so that buffered data is sent to the OS and - # associated Python objects are reclaimed. - for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): - if stream is not None: - stream.flush() - - # Clear assorted module caches. - _path_created.clear() - re.purge() - _strptime._regex_cache.clear() - urllib.parse.clear_cache() - urllib.request.urlcleanup() - linecache.clearcache() - mimetypes._default_mime_types() - filecmp._cache.clear() - struct._clearcache() - doctest.master = None - try: - import ctypes - except ImportError: - # Don't worry about resetting the cache if ctypes is not supported - pass - else: - ctypes._reset_cache() - - # Collect cyclic trash and read memory statistics immediately after. - func1 = sys.getallocatedblocks - func2 = sys.gettotalrefcount - gc.collect() - return func1(), func2() - -def warm_caches(): - # char cache - s = bytes(range(256)) - for i in range(256): - s[i:i+1] - # unicode cache - x = [chr(i) for i in range(256)] - # int cache - x = list(range(-5, 257)) - -def findtestdir(path=None): - return path or os.path.dirname(__file__) or os.curdir - -def removepy(names): - if not names: - return - for idx, name in enumerate(names): - basename, ext = os.path.splitext(name) - if ext == '.py': - names[idx] = basename - -def count(n, word): - if n == 1: - return "%d %s" % (n, word) - else: - return "%d %ss" % (n, word) - -def printlist(x, width=70, indent=4): - """Print the elements of iterable x to stdout. - - Optional arg width (default 70) is the maximum line length. - Optional arg indent (default 4) is the number of blanks with which to - begin each line. - """ - - from textwrap import fill - blanks = ' ' * indent - # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() +from test.libregrtest import main_in_temp_cwd if __name__ == '__main__': diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index cf4b84f..f744127 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,7 +7,8 @@ import faulthandler import getopt import os.path import unittest -from test import regrtest, support, libregrtest +from test import libregrtest +from test import support class ParseArgsTestCase(unittest.TestCase): @@ -15,7 +16,7 @@ class ParseArgsTestCase(unittest.TestCase): def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): - regrtest._parse_args(args) + libregrtest._parse_args(args) self.assertIn(msg, err.getvalue()) def test_help(self): @@ -23,82 +24,82 @@ class ParseArgsTestCase(unittest.TestCase): with self.subTest(opt=opt): with support.captured_stdout() as out, \ self.assertRaises(SystemExit): - regrtest._parse_args([opt]) + libregrtest._parse_args([opt]) self.assertIn('Run Python regression tests.', out.getvalue()) @unittest.skipUnless(hasattr(faulthandler, 'dump_traceback_later'), "faulthandler.dump_traceback_later() required") def test_timeout(self): - ns = regrtest._parse_args(['--timeout', '4.2']) + ns = libregrtest._parse_args(['--timeout', '4.2']) self.assertEqual(ns.timeout, 4.2) self.checkError(['--timeout'], 'expected one argument') self.checkError(['--timeout', 'foo'], 'invalid float value') def test_wait(self): - ns = regrtest._parse_args(['--wait']) + ns = libregrtest._parse_args(['--wait']) self.assertTrue(ns.wait) def test_slaveargs(self): - ns = regrtest._parse_args(['--slaveargs', '[[], {}]']) + ns = libregrtest._parse_args(['--slaveargs', '[[], {}]']) self.assertEqual(ns.slaveargs, '[[], {}]') self.checkError(['--slaveargs'], 'expected one argument') def test_start(self): for opt in '-S', '--start': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.start, 'foo') self.checkError([opt], 'expected one argument') def test_verbose(self): - ns = regrtest._parse_args(['-v']) + ns = libregrtest._parse_args(['-v']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['-vvv']) + ns = libregrtest._parse_args(['-vvv']) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args(['--verbose']) + ns = libregrtest._parse_args(['--verbose']) self.assertEqual(ns.verbose, 1) - ns = regrtest._parse_args(['--verbose'] * 3) + ns = libregrtest._parse_args(['--verbose'] * 3) self.assertEqual(ns.verbose, 3) - ns = regrtest._parse_args([]) + ns = libregrtest._parse_args([]) self.assertEqual(ns.verbose, 0) def test_verbose2(self): for opt in '-w', '--verbose2': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose2) def test_verbose3(self): for opt in '-W', '--verbose3': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.verbose3) def test_quiet(self): for opt in '-q', '--quiet': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_slow(self): for opt in '-o', '--slow': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.print_slow) def test_header(self): - ns = regrtest._parse_args(['--header']) + ns = libregrtest._parse_args(['--header']) self.assertTrue(ns.header) def test_randomize(self): for opt in '-r', '--randomize': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.randomize) def test_randseed(self): - ns = regrtest._parse_args(['--randseed', '12345']) + ns = libregrtest._parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) self.assertTrue(ns.randomize) self.checkError(['--randseed'], 'expected one argument') @@ -107,7 +108,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.fromfile, 'foo') self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo', '-s'], "don't go together") @@ -115,42 +116,42 @@ class ParseArgsTestCase(unittest.TestCase): def test_exclude(self): for opt in '-x', '--exclude': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.exclude) def test_single(self): for opt in '-s', '--single': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'pattern']) + ns = libregrtest._parse_args([opt, 'pattern']) self.assertEqual(ns.match_tests, 'pattern') self.checkError([opt], 'expected one argument') def test_failfast(self): for opt in '-G', '--failfast': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '-v']) + ns = libregrtest._parse_args([opt, '-v']) self.assertTrue(ns.failfast) - ns = regrtest._parse_args([opt, '-W']) + ns = libregrtest._parse_args([opt, '-W']) self.assertTrue(ns.failfast) self.checkError([opt], '-G/--failfast needs either -v or -W') def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'gui,network']) + ns = libregrtest._parse_args([opt, 'gui,network']) self.assertEqual(ns.use_resources, ['gui', 'network']) - ns = regrtest._parse_args([opt, 'gui,none,network']) + ns = libregrtest._parse_args([opt, 'gui,none,network']) self.assertEqual(ns.use_resources, ['network']) expected = list(libregrtest.RESOURCE_NAMES) expected.remove('gui') - ns = regrtest._parse_args([opt, 'all,-gui']) + ns = libregrtest._parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') @@ -158,31 +159,31 @@ class ParseArgsTestCase(unittest.TestCase): def test_memlimit(self): for opt in '-M', '--memlimit': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '4G']) + ns = libregrtest._parse_args([opt, '4G']) self.assertEqual(ns.memlimit, '4G') self.checkError([opt], 'expected one argument') def test_testdir(self): - ns = regrtest._parse_args(['--testdir', 'foo']) + ns = libregrtest._parse_args(['--testdir', 'foo']) self.assertEqual(ns.testdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError(['--testdir'], 'expected one argument') def test_runleaks(self): for opt in '-L', '--runleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.runleaks) def test_huntrleaks(self): for opt in '-R', '--huntrleaks': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, ':']) + ns = libregrtest._parse_args([opt, ':']) self.assertEqual(ns.huntrleaks, (5, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:']) + ns = libregrtest._parse_args([opt, '6:']) self.assertEqual(ns.huntrleaks, (6, 4, 'reflog.txt')) - ns = regrtest._parse_args([opt, ':3']) + ns = libregrtest._parse_args([opt, ':3']) self.assertEqual(ns.huntrleaks, (5, 3, 'reflog.txt')) - ns = regrtest._parse_args([opt, '6:3:leaks.log']) + ns = libregrtest._parse_args([opt, '6:3:leaks.log']) self.assertEqual(ns.huntrleaks, (6, 3, 'leaks.log')) self.checkError([opt], 'expected one argument') self.checkError([opt, '6'], @@ -193,7 +194,7 @@ class ParseArgsTestCase(unittest.TestCase): def test_multiprocess(self): for opt in '-j', '--multiprocess': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '2']) + ns = libregrtest._parse_args([opt, '2']) self.assertEqual(ns.use_mp, 2) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -204,13 +205,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_coverage(self): for opt in '-T', '--coverage': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.trace) def test_coverdir(self): for opt in '-D', '--coverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, 'foo']) + ns = libregrtest._parse_args([opt, 'foo']) self.assertEqual(ns.coverdir, os.path.join(support.SAVEDCWD, 'foo')) self.checkError([opt], 'expected one argument') @@ -218,13 +219,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nocoverdir(self): for opt in '-N', '--nocoverdir': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertIsNone(ns.coverdir) def test_threshold(self): for opt in '-t', '--threshold': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt, '1000']) + ns = libregrtest._parse_args([opt, '1000']) self.assertEqual(ns.threshold, 1000) self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid int value') @@ -232,13 +233,13 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) def test_forever(self): for opt in '-F', '--forever': with self.subTest(opt=opt): - ns = regrtest._parse_args([opt]) + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.forever) @@ -246,26 +247,26 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError(['--xxx'], 'usage:') def test_long_option__partial(self): - ns = regrtest._parse_args(['--qui']) + ns = libregrtest._parse_args(['--qui']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) def test_two_options(self): - ns = regrtest._parse_args(['--quiet', '--exclude']) + ns = libregrtest._parse_args(['--quiet', '--exclude']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertTrue(ns.exclude) def test_option_with_empty_string_value(self): - ns = regrtest._parse_args(['--start', '']) + ns = libregrtest._parse_args(['--start', '']) self.assertEqual(ns.start, '') def test_arg(self): - ns = regrtest._parse_args(['foo']) + ns = libregrtest._parse_args(['foo']) self.assertEqual(ns.args, ['foo']) def test_option_and_arg(self): - ns = regrtest._parse_args(['--quiet', 'foo']) + ns = libregrtest._parse_args(['--quiet', 'foo']) self.assertTrue(ns.quiet) self.assertEqual(ns.verbose, 0) self.assertEqual(ns.args, ['foo']) -- cgit v0.12 From f2b02ced7edfa30e3ee5f15abcdd8f45b36476f5 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 17:47:02 -0700 Subject: Bump up the maximum number of freeblocks --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 487c765..9210e24 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -121,7 +121,7 @@ static PyTypeObject deque_type; added at about the same rate as old blocks are being freed. */ -#define MAXFREEBLOCKS 10 +#define MAXFREEBLOCKS 16 static Py_ssize_t numfreeblocks = 0; static block *freeblocks[MAXFREEBLOCKS]; -- cgit v0.12 From 7c0b70f41910fb70432f32f8ed263a56110a5525 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 21:11:05 -0700 Subject: Minor tweak to the order of variable updates. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 9210e24..66c2a74 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -315,8 +315,8 @@ deque_append(dequeobject *deque, PyObject *item) MARK_END(b->rightlink); deque->rightindex = -1; } - Py_INCREF(item); Py_SIZE(deque)++; + Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; deque_trim_left(deque); @@ -340,8 +340,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) MARK_END(b->leftlink); deque->leftindex = BLOCKLEN; } - Py_INCREF(item); Py_SIZE(deque)++; + Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; deque_trim_right(deque); -- cgit v0.12 From 8299e9b59ecea5cd7236949bf17392e2fd56aec8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Sep 2015 21:31:23 -0700 Subject: Move the copy and clear functions upwards to eliminate unnecessary forward references. --- Modules/_collectionsmodule.c | 230 +++++++++++++++++++++---------------------- 1 file changed, 113 insertions(+), 117 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 66c2a74..5db7aed 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -522,7 +522,40 @@ deque_inplace_concat(dequeobject *deque, PyObject *other) return (PyObject *)deque; } -static PyObject *deque_copy(PyObject *deque); +static PyObject * +deque_copy(PyObject *deque) +{ + dequeobject *old_deque = (dequeobject *)deque; + if (Py_TYPE(deque) == &deque_type) { + dequeobject *new_deque; + PyObject *rv; + + new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); + if (new_deque == NULL) + return NULL; + new_deque->maxlen = old_deque->maxlen; + /* Fast path for the deque_repeat() common case where len(deque) == 1 */ + if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; + rv = deque_append(new_deque, item); + } else { + rv = deque_extend(new_deque, deque); + } + if (rv != NULL) { + Py_DECREF(rv); + return (PyObject *)new_deque; + } + Py_DECREF(new_deque); + return NULL; + } + if (old_deque->maxlen < 0) + return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); + else + return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", + deque, old_deque->maxlen, NULL); +} + +PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); static PyObject * deque_concat(dequeobject *deque, PyObject *other) @@ -552,7 +585,85 @@ deque_concat(dequeobject *deque, PyObject *other) return new_deque; } -static void deque_clear(dequeobject *deque); +static void +deque_clear(dequeobject *deque) +{ + block *b; + block *prevblock; + block *leftblock; + Py_ssize_t leftindex; + Py_ssize_t n; + PyObject *item; + + /* During the process of clearing a deque, decrefs can cause the + deque to mutate. To avoid fatal confusion, we have to make the + deque empty before clearing the blocks and never refer to + anything via deque->ref while clearing. (This is the same + technique used for clearing lists, sets, and dicts.) + + Making the deque empty requires allocating a new empty block. In + the unlikely event that memory is full, we fall back to an + alternate method that doesn't require a new block. Repeating + pops in a while-loop is slower, possibly re-entrant (and a clever + adversary could cause it to never terminate). + */ + + b = newblock(0); + if (b == NULL) { + PyErr_Clear(); + goto alternate_method; + } + + /* Remember the old size, leftblock, and leftindex */ + leftblock = deque->leftblock; + leftindex = deque->leftindex; + n = Py_SIZE(deque); + + /* Set the deque to be empty using the newly allocated block */ + MARK_END(b->leftlink); + MARK_END(b->rightlink); + Py_SIZE(deque) = 0; + deque->leftblock = b; + deque->rightblock = b; + deque->leftindex = CENTER + 1; + deque->rightindex = CENTER; + deque->state++; + + /* Now the old size, leftblock, and leftindex are disconnected from + the empty deque and we can use them to decref the pointers. + */ + while (n--) { + item = leftblock->data[leftindex]; + Py_DECREF(item); + leftindex++; + if (leftindex == BLOCKLEN && n) { + CHECK_NOT_END(leftblock->rightlink); + prevblock = leftblock; + leftblock = leftblock->rightlink; + leftindex = 0; + freeblock(prevblock); + } + } + CHECK_END(leftblock->rightlink); + freeblock(leftblock); + return; + + alternate_method: + while (Py_SIZE(deque)) { + item = deque_pop(deque, NULL); + assert (item != NULL); + Py_DECREF(item); + } +} + +static PyObject * +deque_clearmethod(dequeobject *deque) +{ + deque_clear(deque); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); static PyObject * deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) @@ -1058,77 +1169,6 @@ deque_remove(dequeobject *deque, PyObject *value) PyDoc_STRVAR(remove_doc, "D.remove(value) -- remove first occurrence of value."); -static void -deque_clear(dequeobject *deque) -{ - block *b; - block *prevblock; - block *leftblock; - Py_ssize_t leftindex; - Py_ssize_t n; - PyObject *item; - - /* During the process of clearing a deque, decrefs can cause the - deque to mutate. To avoid fatal confusion, we have to make the - deque empty before clearing the blocks and never refer to - anything via deque->ref while clearing. (This is the same - technique used for clearing lists, sets, and dicts.) - - Making the deque empty requires allocating a new empty block. In - the unlikely event that memory is full, we fall back to an - alternate method that doesn't require a new block. Repeating - pops in a while-loop is slower, possibly re-entrant (and a clever - adversary could cause it to never terminate). - */ - - b = newblock(0); - if (b == NULL) { - PyErr_Clear(); - goto alternate_method; - } - - /* Remember the old size, leftblock, and leftindex */ - leftblock = deque->leftblock; - leftindex = deque->leftindex; - n = Py_SIZE(deque); - - /* Set the deque to be empty using the newly allocated block */ - MARK_END(b->leftlink); - MARK_END(b->rightlink); - Py_SIZE(deque) = 0; - deque->leftblock = b; - deque->rightblock = b; - deque->leftindex = CENTER + 1; - deque->rightindex = CENTER; - deque->state++; - - /* Now the old size, leftblock, and leftindex are disconnected from - the empty deque and we can use them to decref the pointers. - */ - while (n--) { - item = leftblock->data[leftindex]; - Py_DECREF(item); - leftindex++; - if (leftindex == BLOCKLEN && n) { - CHECK_NOT_END(leftblock->rightlink); - prevblock = leftblock; - leftblock = leftblock->rightlink; - leftindex = 0; - freeblock(prevblock); - } - } - CHECK_END(leftblock->rightlink); - freeblock(leftblock); - return; - - alternate_method: - while (Py_SIZE(deque)) { - item = deque_pop(deque, NULL); - assert (item != NULL); - Py_DECREF(item); - } -} - static int valid_index(Py_ssize_t i, Py_ssize_t limit) { @@ -1229,15 +1269,6 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) return 0; } -static PyObject * -deque_clearmethod(dequeobject *deque) -{ - deque_clear(deque); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(clear_doc, "Remove all elements from the deque."); - static void deque_dealloc(dequeobject *deque) { @@ -1277,41 +1308,6 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) } static PyObject * -deque_copy(PyObject *deque) -{ - dequeobject *old_deque = (dequeobject *)deque; - if (Py_TYPE(deque) == &deque_type) { - dequeobject *new_deque; - PyObject *rv; - - new_deque = (dequeobject *)deque_new(&deque_type, (PyObject *)NULL, (PyObject *)NULL); - if (new_deque == NULL) - return NULL; - new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 */ - if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { - PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append(new_deque, item); - } else { - rv = deque_extend(new_deque, deque); - } - if (rv != NULL) { - Py_DECREF(rv); - return (PyObject *)new_deque; - } - Py_DECREF(new_deque); - return NULL; - } - if (old_deque->maxlen < 0) - return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "O", deque, NULL); - else - return PyObject_CallFunction((PyObject *)(Py_TYPE(deque)), "Oi", - deque, old_deque->maxlen, NULL); -} - -PyDoc_STRVAR(copy_doc, "Return a shallow copy of a deque."); - -static PyObject * deque_reduce(dequeobject *deque) { PyObject *dict, *result, *aslist; -- cgit v0.12 From 36b3fbb0ee6e266381cec91a0a3c15fd403c3cfd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 27 Sep 2015 11:19:08 +0200 Subject: Issue #25220: Fix Lib/test/autotest.py --- Lib/test/libregrtest/__init__.py | 2 +- Lib/test/regrtest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index a882ed2..9f7b1c1 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +1,2 @@ from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES -from test.libregrtest.main import main_in_temp_cwd +from test.libregrtest.main import main, main_in_temp_cwd diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index f01cad4..fcc3937 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,7 +11,7 @@ import importlib import os import sys -from test.libregrtest import main_in_temp_cwd +from test.libregrtest import main, main_in_temp_cwd if __name__ == '__main__': -- cgit v0.12 From 8ace8e99b375f978111b62f2cc976ce4f55ecd6b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:26:03 +0300 Subject: Issue #25209: rlcomplete now can add a space or a colon after completed keyword. --- Lib/rlcompleter.py | 6 ++++++ Lib/test/test_rlcompleter.py | 13 +++++++++---- Misc/NEWS | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d517c0e..568cf36 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -106,6 +106,12 @@ class Completer: n = len(text) for word in keyword.kwlist: if word[:n] == text: + if word in {'finally', 'try'}: + word = word + ':' + elif word not in {'False', 'None', 'True', + 'break', 'continue', 'pass', + 'else'}: + word = word + ' ' matches.append(word) for nspace in [builtins.__dict__, self.namespace]: for word, val in nspace.items(): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d37b620..11a8305 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -67,10 +67,15 @@ class TestRlcompleter(unittest.TestCase): def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') - self.assertEqual(completer.complete('a', 0), 'and') - self.assertEqual(completer.complete('a', 1), 'as') - self.assertEqual(completer.complete('as', 2), 'assert') - self.assertEqual(completer.complete('an', 0), 'and') + self.assertEqual(completer.complete('a', 0), 'and ') + self.assertEqual(completer.complete('a', 1), 'as ') + self.assertEqual(completer.complete('as', 2), 'assert ') + self.assertEqual(completer.complete('an', 0), 'and ') + self.assertEqual(completer.complete('pa', 0), 'pass') + self.assertEqual(completer.complete('Fa', 0), 'False') + self.assertEqual(completer.complete('el', 0), 'elif ') + self.assertEqual(completer.complete('el', 1), 'else') + self.assertEqual(completer.complete('tr', 0), 'try:') if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 38eb843..8d4549a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,8 @@ Core and Builtins Library ------- +- Issue #25209: rlcomplete now can add a space or a colon after completed keyword. + - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. - Issue #23517: fromtimestamp() and utcfromtimestamp() methods of -- cgit v0.12 From ab824222d1356ce731f18e9d585d58d84a64637d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Sep 2015 13:43:50 +0300 Subject: Issue #25011: rlcomplete now omits private and special attribute names unless the prefix starts with underscores. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/rlcompleter.py | 35 +++++++++++++++++++++++++---------- Lib/test/test_rlcompleter.py | 15 +++++++++++++++ Misc/NEWS | 3 +++ 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8a2b5d3..0ea3f3b 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -103,6 +103,14 @@ operator (Contributed by Joe Jevnik in :issue:`24379`.) +rlcomplete +---------- + +Private and special attribute names now are omitted unless the prefix starts +with underscores. A space or a colon can be added after completed keyword. +(Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) + + Optimizations ============= diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 568cf36..613848f 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -142,20 +142,35 @@ class Completer: return [] # get the content of the object, except __builtins__ - words = dir(thisobject) - if "__builtins__" in words: - words.remove("__builtins__") + words = set(dir(thisobject)) + words.discard("__builtins__") if hasattr(thisobject, '__class__'): - words.append('__class__') - words.extend(get_class_members(thisobject.__class__)) + words.add('__class__') + words.update(get_class_members(thisobject.__class__)) matches = [] n = len(attr) - for word in words: - if word[:n] == attr and hasattr(thisobject, word): - val = getattr(thisobject, word) - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + if attr == '': + noprefix = '_' + elif attr == '_': + noprefix = '__' + else: + noprefix = None + while True: + for word in words: + if (word[:n] == attr and + not (noprefix and word[:n+1] == noprefix) and + hasattr(thisobject, word)): + val = getattr(thisobject, word) + word = self._callable_postfix(val, "%s.%s" % (expr, word)) + matches.append(word) + if matches or not noprefix: + break + if noprefix == '_': + noprefix = '__' + else: + noprefix = None + matches.sort() return matches def get_class_members(klass): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 11a8305..2ff0788 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -5,6 +5,7 @@ import rlcompleter class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ spam = 1 + _ham = 2 class TestRlcompleter(unittest.TestCase): @@ -51,11 +52,25 @@ class TestRlcompleter(unittest.TestCase): ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) + expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') + for x in dir(None)}) + self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) + self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) # test with a customized namespace self.assertEqual(self.completer.attr_matches('CompleteMe.sp'), ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) + self.assertEqual(self.completer.attr_matches('CompleteMe.'), + ['CompleteMe.mro(', 'CompleteMe.spam']) + self.assertEqual(self.completer.attr_matches('CompleteMe._'), + ['CompleteMe._ham']) + matches = self.completer.attr_matches('CompleteMe.__') + for x in matches: + self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertIn('CompleteMe.__name__', matches) + self.assertIn('CompleteMe.__new__(', matches) CompleteMe.me = CompleteMe self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), diff --git a/Misc/NEWS b/Misc/NEWS index 8d4549a..a44d5c9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -27,6 +27,9 @@ Core and Builtins Library ------- +- Issue #25011: rlcomplete now omits private and special attribute names unless + the prefix starts with underscores. + - Issue #25209: rlcomplete now can add a space or a colon after completed keyword. - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. -- cgit v0.12 From ff6ee25102acbfa61e30a410c3fc75311a3413f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 15:04:11 +0200 Subject: Issue #25122: Remove verbose mode of test_eintr "./python -m test -W test_eintr" wrote Lib/test/eintrdata/eintr_tester.py output to stdout which was not expected. Since test_eintr doesn't hang anymore, remove the verbose mode instead. --- Lib/test/test_eintr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index aabad83..d3cdda0 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,14 +16,7 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - - if support.verbose: - args = [sys.executable, tester] - with subprocess.Popen(args) as proc: - exitcode = proc.wait() - self.assertEqual(exitcode, 0) - else: - script_helper.assert_python_ok(tester) + script_helper.assert_python_ok(tester) if __name__ == "__main__": -- cgit v0.12 From 6b415101a24c26accd409c9e72840953f340a6e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Sep 2015 23:16:17 +0200 Subject: Issue #25220: Add functional tests to test_regrtest * test all available ways to run the Python test suite * test many regrtest options: --slow, --coverage, -r, -u, etc. Note: python -m test --coverage doesn't work on Windows. --- Lib/test/test_regrtest.py | 293 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 291 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f744127..c0f1e7b 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1,18 +1,33 @@ """ Tests of regrtest.py. + +Note: test_regrtest cannot be run twice in parallel. """ import argparse import faulthandler import getopt import os.path +import platform +import re +import subprocess +import sys +import textwrap import unittest from test import libregrtest from test import support +from test.support import script_helper -class ParseArgsTestCase(unittest.TestCase): - """Test regrtest's argument parsing.""" +Py_DEBUG = hasattr(sys, 'getobjects') +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') +ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) + + +class ParseArgsTestCase(unittest.TestCase): + """ + Test regrtest's argument parsing, function _parse_args(). + """ def checkError(self, args, msg): with support.captured_stderr() as err, self.assertRaises(SystemExit): @@ -272,5 +287,279 @@ class ParseArgsTestCase(unittest.TestCase): self.assertEqual(ns.args, ['foo']) +class BaseTestCase(unittest.TestCase): + TEST_UNIQUE_ID = 1 + TESTNAME_PREFIX = 'test_regrtest_' + TESTNAME_REGEX = r'test_[a-z0-9_]+' + + def setUp(self): + self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + + # When test_regrtest is interrupted by CTRL+c, it can leave + # temporary test files + remove = [entry.path + for entry in os.scandir(self.testdir) + if (entry.name.startswith(self.TESTNAME_PREFIX) + and entry.name.endswith(".py"))] + for path in remove: + print("WARNING: test_regrtest: remove %s" % path) + support.unlink(path) + + def create_test(self, name=None, code=''): + if not name: + name = 'noop%s' % BaseTestCase.TEST_UNIQUE_ID + BaseTestCase.TEST_UNIQUE_ID += 1 + + # test_regrtest cannot be run twice in parallel because + # of setUp() and create_test() + name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) + path = os.path.join(self.testdir, name + '.py') + + self.addCleanup(support.unlink, path) + # Use 'x' mode to ensure that we do not override existing tests + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + return name + + def regex_search(self, regex, output): + match = re.search(regex, output, re.MULTILINE) + if not match: + self.fail("%r not found in %r" % (regex, output)) + return match + + def check_line(self, output, regex): + regex = re.compile(r'^' + regex, re.MULTILINE) + self.assertRegex(output, regex) + + def parse_executed_tests(self, output): + parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, + output, + re.MULTILINE) + return set(match.group(1) for match in parser) + + def check_executed_tests(self, output, tests, skipped=None): + if isinstance(tests, str): + tests = [tests] + executed = self.parse_executed_tests(output) + self.assertEqual(executed, set(tests), output) + ntest = len(tests) + if skipped: + if isinstance(skipped, str): + skipped = [skipped] + nskipped = len(skipped) + + plural = 's' if nskipped != 1 else '' + names = ' '.join(sorted(skipped)) + expected = (r'%s test%s skipped:\n %s$' + % (nskipped, plural, names)) + self.check_line(output, expected) + + ok = ntest - nskipped + if ok: + self.check_line(output, r'%s test OK\.$' % ok) + else: + self.check_line(output, r'All %s tests OK\.$' % ntest) + + def parse_random_seed(self, output): + match = self.regex_search(r'Using random seed ([0-9]+)', output) + randseed = int(match.group(1)) + self.assertTrue(0 <= randseed <= 10000000, randseed) + return randseed + + +class ProgramsTestCase(BaseTestCase): + """ + Test various ways to run the Python test suite. Use options close + to options used on the buildbot. + """ + + NTEST = 4 + + def setUp(self): + super().setUp() + + # Create NTEST tests doing nothing + self.tests = [self.create_test() for index in range(self.NTEST)] + + self.python_args = ['-Wd', '-E', '-bb'] + self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + if sys.platform == 'win32': + self.regrtest_args.append('-n') + + def check_output(self, output): + self.parse_random_seed(output) + self.check_executed_tests(output, self.tests) + + def run_tests(self, args): + res = script_helper.assert_python_ok(*args) + output = os.fsdecode(res.out) + self.check_output(output) + + def test_script_regrtest(self): + # Lib/test/regrtest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_test(self): + # -m test + args = [*self.python_args, '-m', 'test', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_regrtest(self): + # -m test.regrtest + args = [*self.python_args, '-m', 'test.regrtest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_autotest(self): + # -m test.autotest + args = [*self.python_args, '-m', 'test.autotest', + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_module_from_test_autotest(self): + # from test import autotest + code = 'from test import autotest' + args = [*self.python_args, '-c', code, + *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_script_autotest(self): + # Lib/test/autotest.py + script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + args = [*self.python_args, script, *self.regrtest_args, *self.tests] + self.run_tests(args) + + def test_tools_script_run_tests(self): + # Tools/scripts/run_tests.py + script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') + self.run_tests([script, *self.tests]) + + def run_rt_bat(self, script, *args): + rt_args = [] + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build + + args = [script, *rt_args, *args] + proc = subprocess.run(args, + check=True, universal_newlines=True, + input='', + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.check_output(proc.stdout) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_tools_buildbot_test(self): + # Tools\buildbot\test.bat + script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') + self.run_rt_bat(script, *self.tests) + + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + def test_pcbuild_rt(self): + # PCbuild\rt.bat + script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') + # -q: quick, don't run tests twice + rt_args = ["-q"] + self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + + +class ArgsTestCase(BaseTestCase): + """ + Test arguments of the Python test suite. + """ + + def run_tests(self, *args): + args = ['-m', 'test', *args] + res = script_helper.assert_python_ok(*args) + return os.fsdecode(res.out) + + def test_resources(self): + # test -u command line option + tests = {} + for resource in ('audio', 'network'): + code = 'from test import support\nsupport.requires(%r)' % resource + tests[resource] = self.create_test(resource, code) + test_names = sorted(tests.values()) + + # -u all: 2 resources enabled + output = self.run_tests('-u', 'all', *test_names) + self.check_executed_tests(output, test_names) + + # -u audio: 1 resource enabled + output = self.run_tests('-uaudio', *test_names) + self.check_executed_tests(output, test_names, + skipped=tests['network']) + + # no option: 0 resources enabled + output = self.run_tests(*test_names) + self.check_executed_tests(output, test_names, + skipped=test_names) + + def test_random(self): + # test -r and --randseed command line option + code = textwrap.dedent(""" + import random + print("TESTRANDOM: %s" % random.randint(1, 1000)) + """) + test = self.create_test('random', code) + + # first run to get the output with the random seed + output = self.run_tests('-r', test) + randseed = self.parse_random_seed(output) + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random = int(match.group(1)) + + # try to reproduce with the random seed + output = self.run_tests('-r', '--randseed=%s' % randseed, test) + randseed2 = self.parse_random_seed(output) + self.assertEqual(randseed2, randseed) + + match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output) + test_random2 = int(match.group(1)) + self.assertEqual(test_random2, test_random) + + def test_fromfile(self): + # test --fromfile + tests = [self.create_test() for index in range(5)] + + # Write the list of files using a format similar to regrtest output: + # [1/2] test_1 + # [2/2] test_2 + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + for index, name in enumerate(tests, 1): + print("[%s/%s] %s" % (index, len(tests), name), file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + def test_slow(self): + # test --slow + tests = [self.create_test() for index in range(3)] + output = self.run_tests("--slow", *tests) + self.check_executed_tests(output, tests) + regex = ('10 slowest tests:\n' + '(?:%s: [0-9]+\.[0-9]+s\n){%s}' + % (self.TESTNAME_REGEX, len(tests))) + self.check_line(output, regex) + + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") + def test_coverage(self): + # test --coverage + test = self.create_test() + output = self.run_tests("--coverage", test) + executed = self.parse_executed_tests(output) + self.assertEqual(executed, {test}, output) + regex = ('lines +cov% +module +\(path\)\n' + '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + self.check_line(output, regex) + + if __name__ == '__main__': unittest.main() -- cgit v0.12 From e143c1f94f923db17a0f6d17d05e4af440178c1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 01:02:37 +0200 Subject: Fix test_regrtest.test_tools_buildbot_test() Issue #25220: Fix test_regrtest.test_tools_buildbot_test() on release build (on Windows), pass "+d" option to test.bat. --- Lib/test/test_regrtest.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c0f1e7b..ca4b356 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -438,14 +438,7 @@ class ProgramsTestCase(BaseTestCase): script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') self.run_tests([script, *self.tests]) - def run_rt_bat(self, script, *args): - rt_args = [] - if platform.architecture()[0] == '64bit': - rt_args.append('-x64') # 64-bit build - if Py_DEBUG: - rt_args.append('-d') # Debug build - - args = [script, *rt_args, *args] + def run_batch(self, *args): proc = subprocess.run(args, check=True, universal_newlines=True, input='', @@ -456,15 +449,23 @@ class ProgramsTestCase(BaseTestCase): def test_tools_buildbot_test(self): # Tools\buildbot\test.bat script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') - self.run_rt_bat(script, *self.tests) + test_args = [] + if platform.architecture()[0] == '64bit': + test_args.append('-x64') # 64-bit build + if not Py_DEBUG: + test_args.append('+d') # Release build, use python.exe + self.run_batch(script, *test_args, *self.tests) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_pcbuild_rt(self): # PCbuild\rt.bat script = os.path.join(ROOT_DIR, r'PCbuild\rt.bat') - # -q: quick, don't run tests twice - rt_args = ["-q"] - self.run_rt_bat(script, *rt_args, *self.regrtest_args, *self.tests) + rt_args = ["-q"] # Quick, don't run tests twice + if platform.architecture()[0] == '64bit': + rt_args.append('-x64') # 64-bit build + if Py_DEBUG: + rt_args.append('-d') # Debug build, use python_d.exe + self.run_batch(script, *rt_args, *self.regrtest_args, *self.tests) class ArgsTestCase(BaseTestCase): -- cgit v0.12 From c3713e9706e51bbd30958c27d35e7fda764b0c4a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 12:32:13 +0200 Subject: Optimize ascii/latin1+surrogateescape encoders Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` error handler: the encoders are now up to 3 times as fast. Initial patch written by Serhiy Storchaka. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/test/test_codecs.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ Objects/unicodeobject.c | 16 +++++++++++++ 4 files changed, 83 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0ea3f3b..16cdca0 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -117,6 +117,9 @@ Optimizations * The ASCII decoder is now up to 60 times as fast for error handlers: ``surrogateescape``, ``ignore`` and ``replace``. +* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error + error ``surrogateescape``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index e0e3119..254c0c1 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3060,7 +3060,31 @@ class CodePageTest(unittest.TestCase): class ASCIITest(unittest.TestCase): + def test_encode(self): + self.assertEqual('abc123'.encode('ascii'), b'abc123') + + def test_encode_error(self): + for data, error_handler, expected in ( + ('[\x80\xff\u20ac]', 'ignore', b'[]'), + ('[\x80\xff\u20ac]', 'replace', b'[???]'), + ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), + ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('ascii', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\xff'.encode('ascii', 'surrogateescape') + def test_decode(self): + self.assertEqual(b'abc'.decode('ascii'), 'abc') + + def test_decode_error(self): for data, error_handler, expected in ( (b'[\x80\xff]', 'ignore', '[]'), (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), @@ -3073,5 +3097,41 @@ class ASCIITest(unittest.TestCase): expected) +class Latin1Test(unittest.TestCase): + def test_encode(self): + for data, expected in ( + ('abc', b'abc'), + ('\x80\xe9\xff', b'\x80\xe9\xff'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.encode('latin1'), expected) + + def test_encode_errors(self): + for data, error_handler, expected in ( + ('[\u20ac\udc80]', 'ignore', b'[]'), + ('[\u20ac\udc80]', 'replace', b'[??]'), + ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), + ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.encode('latin1', error_handler), + expected) + + def test_encode_surrogateescape_error(self): + with self.assertRaises(UnicodeEncodeError): + # the first character can be decoded, but not the second + '\udc80\u20ac'.encode('latin1', 'surrogateescape') + + def test_decode(self): + for data, expected in ( + (b'abc', 'abc'), + (b'[\x80\xff]', '[\x80\xff]'), + ): + with self.subTest(data=data, expected=expected): + self.assertEqual(data.decode('latin1'), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 72e521d..ceac0a3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25227: Optimize ASCII and latin1 encoders with the ``surrogateescape`` + error handler: the encoders are now up to 3 times as fast. Initial patch + written by Serhiy Storchaka. + - Issue #25003: On Solaris 11.3 or newer, os.urandom() now uses the getrandom() function instead of the getentropy() function. The getentropy() function is blocking to generate very good quality entropy, os.urandom() diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index da2aac7..6657cd4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6532,6 +6532,22 @@ unicode_encode_ucs1(PyObject *unicode, pos = collend; break; + case _Py_ERROR_SURROGATEESCAPE: + for (i = collstart; i < collend; ++i) { + ch = PyUnicode_READ(kind, data, i); + if (ch < 0xdc80 || 0xdcff < ch) { + /* Not a UTF-8b surrogate */ + break; + } + *str++ = (char)(ch - 0xdc00); + ++pos; + } + if (i >= collend) + break; + collstart = pos; + assert(collstart != collend); + /* fallback to general error handling */ + default: repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, encoding, reason, unicode, &exc, -- cgit v0.12 From 2bfed53b88cea7a4ec4b45cac33c62d9342796bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:41:46 +0200 Subject: Try to fix _PyTime_AsTimevalStruct_impl() on OpenBSD It looks like the check for integer overflow doesn't work on x86 OpenBSD 5.8. --- Python/pytime.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9889a3b..53611b1 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs; + _PyTime_t secs, secs2; int us; int res; @@ -467,7 +467,8 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + secs2 = (_PyTime_t)tv->tv_sec; + if (res < 0 || secs2 != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From 3f7468507ae83d66cee059fb8e51692d8adb797b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:47:15 +0200 Subject: test --- Lib/test/libregrtest/main.py | 4 +--- Lib/test/test_regrtest.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cff..3724f27 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,9 +246,7 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + tracer = trace.Trace(trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356..5906c17 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,8 +549,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From 449b27179931aa570597556ed4865ac2f1f981c9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 13:59:50 +0200 Subject: Issue #18174: Explain why is_valid_fd() uses dup() instead of fstat() --- Python/pylifecycle.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4f5efc9..857a543 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -972,6 +972,9 @@ is_valid_fd(int fd) if (fd < 0 || !_PyVerify_fd(fd)) return 0; _Py_BEGIN_SUPPRESS_IPH + /* Prefer dup() over fstat(). fstat() can require input/output whereas + dup() doesn't, there is a low risk of EMFILE/ENFILE at Python + startup. */ fd2 = dup(fd); if (fd2 >= 0) close(fd2); -- cgit v0.12 From feabaed054ae94cf1104d21f2c13568c5c8b6622 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:02:35 +0200 Subject: Oops, revert unwanted change, sorry --- Lib/test/libregrtest/main.py | 4 +++- Lib/test/test_regrtest.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 3724f27..5d34cff 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -246,7 +246,9 @@ def main(tests=None, **kwargs): random.shuffle(selected) if ns.trace: import trace, tempfile - tracer = trace.Trace(trace=False, count=True) + tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 5906c17..ca4b356 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -549,6 +549,8 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + @unittest.skipIf(sys.platform == 'win32', + "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From 427713403546696d1171905095089dccef39c54c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 14:17:09 +0200 Subject: Issue #25220: Add test for --wait in test_regrtest Replace script_helper.assert_python_ok() with subprocess.run(). --- Lib/test/test_regrtest.py | 48 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ca4b356..54bbfe4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -16,7 +16,6 @@ import textwrap import unittest from test import libregrtest from test import support -from test.support import script_helper Py_DEBUG = hasattr(sys, 'getobjects') @@ -366,6 +365,31 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed + def run_command(self, args, input=None): + if not input: + input = '' + try: + return subprocess.run(args, + check=True, universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as exc: + self.fail("%s\n" + "\n" + "stdout:\n" + "%s\n" + "\n" + "stderr:\n" + "%s" + % (str(exc), exc.stdout, exc.stderr)) + + + def run_python(self, args, **kw): + args = [sys.executable, '-X', 'faulthandler', '-I', *args] + proc = self.run_command(args, **kw) + return proc.stdout + class ProgramsTestCase(BaseTestCase): """ @@ -391,9 +415,8 @@ class ProgramsTestCase(BaseTestCase): self.check_executed_tests(output, self.tests) def run_tests(self, args): - res = script_helper.assert_python_ok(*args) - output = os.fsdecode(res.out) - self.check_output(output) + stdout = self.run_python(args) + self.check_output(stdout) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -439,10 +462,7 @@ class ProgramsTestCase(BaseTestCase): self.run_tests([script, *self.tests]) def run_batch(self, *args): - proc = subprocess.run(args, - check=True, universal_newlines=True, - input='', - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = self.run_command(args) self.check_output(proc.stdout) @unittest.skipUnless(sys.platform == 'win32', 'Windows only') @@ -473,10 +493,8 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args): - args = ['-m', 'test', *args] - res = script_helper.assert_python_ok(*args) - return os.fsdecode(res.out) + def run_tests(self, *args, input=None): + return self.run_python(['-m', 'test', *args], input=input) def test_resources(self): # test -u command line option @@ -561,6 +579,12 @@ class ArgsTestCase(BaseTestCase): '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) + def test_wait(self): + # test --wait + test = self.create_test() + output = self.run_tests("--wait", test, input='key') + self.check_line(output, 'Press any key to continue') + if __name__ == '__main__': unittest.main() -- cgit v0.12 From dad20e4876b339a559d61810299f96712719687e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 22:48:52 +0200 Subject: Issue #25220: Split the huge main() function of libregrtest.main into a class with attributes and methods. The --threshold command line option is now ignored if the gc module is missing. * Convert main() variables to Regrtest attributes, document some attributes * Convert accumulate_result() function to a method * Create setup_python() function and setup_regrtest() method. * Import gc at top level * Move resource.setrlimit() and the code to make the module paths absolute into the new setup_python() function. So this code is no more executed when the module is imported, only when main() is executed. We have a better control on when the setup is done. * Move textwrap import from printlist() to the top level. * Some other minor cleanup. --- Lib/test/libregrtest/main.py | 701 ++++++++++++++++++++++++------------------- 1 file changed, 387 insertions(+), 314 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 5d34cff..63388c9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,13 +1,14 @@ import faulthandler import json import os +import platform +import random import re +import signal import sys -import tempfile import sysconfig -import signal -import random -import platform +import tempfile +import textwrap import traceback import unittest from test.libregrtest.runtest import ( @@ -19,45 +20,15 @@ from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support try: + import gc +except ImportError: + gc = None +try: import threading except ImportError: threading = None -# Some times __path__ and __file__ are not absolute (e.g. while running from -# Lib/) and, if we change the CWD to run the tests in a temporary dir, some -# imports might fail. This affects only the modules imported before os.chdir(). -# These modules are searched first in sys.path[0] (so '' -- the CWD) and if -# they are found in the CWD their __file__ and __path__ will be relative (this -# happens before the chdir). All the modules imported after the chdir, are -# not found in the CWD, and since the other paths in sys.path[1:] are absolute -# (site.py absolutize them), the __file__ and __path__ will be absolute too. -# Therefore it is necessary to absolutize manually the __file__ and __path__ of -# the packages to prevent later imports to fail when the CWD is different. -for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - -# MacOSX (a.k.a. Darwin) has a default stack size that is too small -# for deeply recursive regular expressions. We see this as crashes in -# the Python test suite when running test_re.py and test_sre.py. The -# fix is to set the stack limit to 2048. -# This approach may also be useful for other Unixy platforms that -# suffer from small default stack limits. -if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - # When tests are run from the Python build directory, it is best practice # to keep the test files in a subfolder. This eases the cleanup of leftover # files using the "make distclean" command. @@ -68,7 +39,73 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def main(tests=None, **kwargs): +def slave_runner(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +def setup_python(): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + +class Regrtest: """Execute a test suite. This also parses command-line options and modifies its behavior @@ -91,210 +128,257 @@ def main(tests=None, **kwargs): directly to set the values that would normally be set by flags on the command line. """ - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) + def __init__(self): + # Namespace of command line options + self.ns = None + + # tests + self.tests = [] + self.selected = [] + + # test results + self.good = [] + self.bad = [] + self.skipped = [] + self.resource_denieds = [] + self.environment_changed = [] + self.interrupted = False - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) + # used by --slow + self.test_times = [] - replace_stdout() + # used by --coverage, trace.Trace instance + self.tracer = None - support.record_original_stdout(sys.stdout) + # used by --findleaks, store for gc.garbage + self.found_garbage = [] - ns = _parse_args(sys.argv[1:], **kwargs) - - if ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - if ns.threshold is not None: - import gc - gc.set_threshold(ns.threshold) - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if ns.wait: - input("Press any key to continue...") - - if ns.slaveargs is not None: - args, kwargs = json.loads(ns.slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - good = [] - bad = [] - skipped = [] - resource_denieds = [] - environment_changed = [] - interrupted = False - - if ns.findleaks: - try: - import gc - except ImportError: - print('No GC available, disabling findleaks.') - ns.findleaks = False - else: - # Uncomment the line below to report garbage that is not - # freeable by reference counting alone. By default only - # garbage that is not collectable by the GC is reported. - #gc.set_debug(gc.DEBUG_SAVEALL) - found_garbage = [] - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + # used to display the progress bar "[ 3/100]" + self.test_count = '' + self.test_count_width = 1 - if ns.single: - filename = os.path.join(TEMPDIR, 'pynexttest') - try: - with open(filename, 'r') as fp: - next_test = fp.read().strip() - tests = [next_test] - except OSError: - pass + # used by --single + self.next_single_test = None + self.next_single_filename = None - if ns.fromfile: - tests = [] - with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') - for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - tests.extend(guts) - - # Strip .py extensions. - removepy(ns.args) - removepy(tests) - - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() - if ns.exclude: - for arg in ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) - ns.args = [] - - # For a partial run, we do not need to clutter the output. - if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - - # if testdir is set, then we are not running the python tests suite, so - # don't add default tests to be executed or skipped (pass empty values) - if ns.testdir: - alltests = findtests(ns.testdir, list(), set()) - else: - alltests = findtests(ns.testdir, stdtests, nottests) - - selected = tests or ns.args or alltests - if ns.single: - selected = selected[:1] - try: - next_single_test = alltests[alltests.index(selected[0])+1] - except IndexError: - next_single_test = None - # Remove all the selected tests that precede start if it's set. - if ns.start: - try: - del selected[:selected.index(ns.start)] - except ValueError: - print("Couldn't find starting test (%s), using all tests" % ns.start) - if ns.randomize: - if ns.random_seed is None: - ns.random_seed = random.randrange(10000000) - random.seed(ns.random_seed) - print("Using random seed", ns.random_seed) - random.shuffle(selected) - if ns.trace: - import trace, tempfile - tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - - test_times = [] - support.verbose = ns.verbose # Tell tests to be moderately quiet - support.use_resources = ns.use_resources - save_modules = sys.modules.keys() - - def accumulate_result(test, result): + def accumulate_result(self, test, result): ok, test_time = result - test_times.append((test_time, test)) + self.test_times.append((test_time, test)) if ok == PASSED: - good.append(test) + self.good.append(test) elif ok == FAILED: - bad.append(test) + self.bad.append(test) elif ok == ENV_CHANGED: - environment_changed.append(test) + self.environment_changed.append(test) elif ok == SKIPPED: - skipped.append(test) + self.skipped.append(test) elif ok == RESOURCE_DENIED: - skipped.append(test) - resource_denieds.append(test) - - if ns.forever: - def test_forever(tests=list(selected)): - while True: - for test in tests: - yield test - if bad: - return - tests = test_forever() - test_count = '' - test_count_width = 3 - else: - tests = iter(selected) - test_count = '/{}'.format(len(selected)) - test_count_width = len(test_count) - 1 + self.skipped.append(test) + self.resource_denieds.append(test) + + def display_progress(self, test_index, test): + if self.ns.quiet: + return + fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" + print(fmt.format( + self.test_count_width, test_index, self.test_count, len(self.bad), test)) + sys.stdout.flush() + + def setup_regrtest(self): + if self.ns.huntrleaks: + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() - if ns.use_mp: + if self.ns.memlimit is not None: + support.set_memlimit(self.ns.memlimit) + + if self.ns.threshold is not None: + if gc is not None: + gc.set_threshold(self.ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if self.ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + if self.ns.findleaks: + if gc is not None: + # Uncomment the line below to report garbage that is not + # freeable by reference counting alone. By default only + # garbage that is not collectable by the GC is reported. + pass + #gc.set_debug(gc.DEBUG_SAVEALL) + else: + print('No GC available, disabling --findleaks') + self.ns.findleaks = False + + if self.ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Strip .py extensions. + removepy(self.ns.args) + + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + + def find_tests(self, tests): + self.tests = tests + + if self.ns.single: + self.next_single_filename = os.path.join(TEMPDIR, 'pynexttest') + try: + with open(self.next_single_filename, 'r') as fp: + next_test = fp.read().strip() + self.tests = [next_test] + except OSError: + pass + + if self.ns.fromfile: + self.tests = [] + with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: + count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') + for line in fp: + line = count_pat.sub('', line) + guts = line.split() # assuming no test has whitespace in its name + if guts and not guts[0].startswith('#'): + self.tests.extend(guts) + + removepy(self.tests) + + stdtests = STDTESTS[:] + nottests = NOTTESTS.copy() + if self.ns.exclude: + for arg in self.ns.args: + if arg in stdtests: + stdtests.remove(arg) + nottests.add(arg) + self.ns.args = [] + + # For a partial run, we do not need to clutter the output. + if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if self.ns.testdir: + alltests = findtests(self.ns.testdir, list(), set()) + else: + alltests = findtests(self.ns.testdir, stdtests, nottests) + + self.selected = self.tests or self.ns.args or alltests + if self.ns.single: + self.selected = self.selected[:1] + try: + pos = alltests.index(self.selected[0]) + self.next_single_test = alltests[pos + 1] + except IndexError: + pass + + # Remove all the self.selected tests that precede start if it's set. + if self.ns.start: + try: + del self.selected[:self.selected.index(self.ns.start)] + except ValueError: + print("Couldn't find starting test (%s), using all tests" % self.ns.start) + + if self.ns.randomize: + if self.ns.random_seed is None: + self.ns.random_seed = random.randrange(10000000) + random.seed(self.ns.random_seed) + print("Using random seed", self.ns.random_seed) + random.shuffle(self.selected) + + def display_result(self): + if self.interrupted: + # print a newline after ^C + print() + print("Test suite interrupted by signal SIGINT.") + omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + print(count(len(omitted), "test"), "omitted:") + printlist(omitted) + + if self.good and not self.ns.quiet: + if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + print("All", end=' ') + print(count(len(self.good), "test"), "OK.") + + if self.ns.print_slow: + self.test_times.sort(reverse=True) + print("10 slowest tests:") + for time, test in self.test_times[:10]: + print("%s: %.1fs" % (test, time)) + + if self.bad: + print(count(len(self.bad), "test"), "failed:") + printlist(self.bad) + + if self.environment_changed: + print("{} altered the execution environment:".format( + count(len(self.environment_changed), "test"))) + printlist(self.environment_changed) + + if self.skipped and not self.ns.quiet: + print(count(len(self.skipped), "test"), "skipped:") + printlist(self.skipped) + + if self.ns.verbose2 and self.bad: + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test) + sys.stdout.flush() + try: + self.ns.verbose = True + ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, + timeout=self.ns.timeout) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + + def _run_tests_mp(self): try: from threading import Thread except ImportError: print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue + debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() - pending = MultiprocessTests(tests) + pending = MultiprocessTests(self.tests) + def work(): # A worker thread. try: @@ -304,7 +388,7 @@ def main(tests=None, **kwargs): except StopIteration: output.put((None, None, None, None)) return - retcode, stdout, stderr = run_test_in_subprocess(test, ns) + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) @@ -321,23 +405,20 @@ def main(tests=None, **kwargs): except BaseException: output.put((None, None, None, None)) raise - workers = [Thread(target=work) for i in range(ns.use_mp)] + + workers = [Thread(target=work) for i in range(self.ns.use_mp)] for worker in workers: worker.start() finished = 0 test_index = 1 try: - while finished < ns.use_mp: + while finished < self.ns.use_mp: test, stdout, stderr, result = output.get() if test is None: finished += 1 continue - accumulate_result(test, result) - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, - len(bad), test)) + self.accumulate_result(test, result) + self.display_progress(test_index, test) if stdout: print(stdout) if stderr: @@ -350,110 +431,99 @@ def main(tests=None, **kwargs): raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: - interrupted = True + self.interrupted = True pending.interrupted = True for worker in workers: worker.join() - else: - for test_index, test in enumerate(tests, 1): - if not ns.quiet: - fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - test_count_width, test_index, test_count, len(bad), test)) - sys.stdout.flush() - if ns.trace: + + def _run_tests_sequential(self): + save_modules = sys.modules.keys() + + for test_index, test in enumerate(self.tests, 1): + self.display_progress(test_index, test) + if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', - globals=globals(), locals=vars()) + cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, ns.verbose, ns.quiet, - ns.huntrleaks, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests) - accumulate_result(test, result) + result = runtest(test, self.ns.verbose, self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) except KeyboardInterrupt: - interrupted = True + self.interrupted = True break - if ns.findleaks: + if self.ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again - found_garbage.extend(gc.garbage) + self.found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) - if interrupted: - # print a newline after ^C - print() - print("Test suite interrupted by signal SIGINT.") - omitted = set(selected) - set(good) - set(bad) - set(skipped) - print(count(len(omitted), "test"), "omitted:") - printlist(omitted) - if good and not ns.quiet: - if not bad and not skipped and not interrupted and len(good) > 1: - print("All", end=' ') - print(count(len(good), "test"), "OK.") - if ns.print_slow: - test_times.sort(reverse=True) - print("10 slowest tests:") - for time, test in test_times[:10]: - print("%s: %.1fs" % (test, time)) - if bad: - print(count(len(bad), "test"), "failed:") - printlist(bad) - if environment_changed: - print("{} altered the execution environment:".format( - count(len(environment_changed), "test"))) - printlist(environment_changed) - if skipped and not ns.quiet: - print(count(len(skipped), "test"), "skipped:") - printlist(skipped) - - if ns.verbose2 and bad: - print("Re-running failed tests in verbose mode") - for test in bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() - try: - ns.verbose = True - ok = runtest(test, True, ns.quiet, ns.huntrleaks, - timeout=ns.timeout) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - bad.remove(test) - else: - if bad: - print(count(len(bad), 'test'), "failed again:") - printlist(bad) - - if ns.single: - if next_single_test: - with open(filename, 'w') as fp: - fp.write(next_single_test + '\n') - else: - os.unlink(filename) + def run_tests(self): + support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.use_resources = self.ns.use_resources - if ns.trace: - r = tracer.results() - r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) + if self.ns.forever: + def test_forever(tests): + while True: + for test in tests: + yield test + if self.bad: + return + self.tests = test_forever(list(self.selected)) + self.test_count = '' + self.test_count_width = 3 + else: + self.tests = iter(self.selected) + self.test_count = '/{}'.format(len(self.selected)) + self.test_count_width = len(self.test_count) - 1 - if ns.runleaks: - os.system("leaks %d" % os.getpid()) + if self.ns.use_mp: + self._run_tests_mp() + else: + self._run_tests_sequential() - sys.exit(len(bad) > 0 or interrupted) + def finalize(self): + if self.next_single_filename: + if self.next_single_test: + with open(self.next_single_filename, 'w') as fp: + fp.write(self.next_single_test + '\n') + else: + os.unlink(self.next_single_filename) + + if self.ns.trace: + r = self.tracer.results() + r.write_results(show_missing=True, summary=True, + coverdir=self.ns.coverdir) + + if self.ns.runleaks: + os.system("leaks %d" % os.getpid()) + + def main(self, tests=None, **kwargs): + setup_python() + self.ns = _parse_args(sys.argv[1:], **kwargs) + self.setup_regrtest() + if self.ns.wait: + input("Press any key to continue...") + if self.ns.slaveargs is not None: + slave_runner(self.ns.slaveargs) + self.find_tests(tests) + self.run_tests() + self.display_result() + self.finalize() + sys.exit(len(self.bad) > 0 or self.interrupted) # We do not use a generator so multiple threads can call next(). @@ -518,11 +588,14 @@ def printlist(x, width=70, indent=4): begin each line. """ - from textwrap import fill blanks = ' ' * indent # Print the sorted list: 'x' may be a '--random' list or a set() - print(fill(' '.join(str(elt) for elt in sorted(x)), width, - initial_indent=blanks, subsequent_indent=blanks)) + print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, + initial_indent=blanks, subsequent_indent=blanks)) + + +def main(tests=None, **kwargs): + Regrtest().main(tests=tests, **kwargs) def main_in_temp_cwd(): -- cgit v0.12 From 56e05dd0b03386b2da879283f6ab165f857417be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:15:38 +0200 Subject: Issue #25220: Create libregrtest/runtest_mp.py Move the code to run tests in multiple processes using threading and subprocess to a new submodule. Move also slave_runner() (renamed to run_tests_slave()) and run_test_in_subprocess() (renamed to run_tests_in_subprocess()) there. --- Lib/test/libregrtest/main.py | 120 ++-------------------------- Lib/test/libregrtest/runtest.py | 33 -------- Lib/test/libregrtest/runtest_mp.py | 158 +++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 147 deletions(-) create mode 100644 Lib/test/libregrtest/runtest_mp.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 63388c9..b66f045 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,4 @@ import faulthandler -import json import os import platform import random @@ -9,13 +8,10 @@ import sys import sysconfig import tempfile import textwrap -import traceback import unittest from test.libregrtest.runtest import ( - findtests, runtest, run_test_in_subprocess, - STDTESTS, NOTTESTS, - PASSED, FAILED, ENV_CHANGED, SKIPPED, - RESOURCE_DENIED, INTERRUPTED, CHILD_ERROR) + findtests, runtest, + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args from test import support @@ -39,23 +35,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def slave_runner(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): - unittest.BaseTestSuite._cleanup = False - try: - result = runtest(*args, **kwargs) - except KeyboardInterrupt: - result = INTERRUPTED, '' - except BaseException as e: - traceback.print_exc() - result = CHILD_ERROR, str(e) - sys.stdout.flush() - print() # Force a newline (just in case) - print(json.dumps(result)) - sys.exit(0) - - def setup_python(): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -367,75 +346,6 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_mp(self): - try: - from threading import Thread - except ImportError: - print("Multiprocess option requires thread support") - sys.exit(2) - from queue import Queue - - debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - output = Queue() - pending = MultiprocessTests(self.tests) - - def work(): - # A worker thread. - try: - while True: - try: - test = next(pending) - except StopIteration: - output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - return - if not result: - output.put((None, None, None, None)) - return - result = json.loads(result) - output.put((test, stdout.rstrip(), stderr.rstrip(), result)) - except BaseException: - output.put((None, None, None, None)) - raise - - workers = [Thread(target=work) for i in range(self.ns.use_mp)] - for worker in workers: - worker.start() - finished = 0 - test_index = 1 - try: - while finished < self.ns.use_mp: - test, stdout, stderr, result = output.get() - if test is None: - finished += 1 - continue - self.accumulate_result(test, result) - self.display_progress(test_index, test) - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() - if result[0] == INTERRUPTED: - raise KeyboardInterrupt - if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) - test_index += 1 - except KeyboardInterrupt: - self.interrupted = True - pending.interrupted = True - for worker in workers: - worker.join() - def _run_tests_sequential(self): save_modules = sys.modules.keys() @@ -491,7 +401,8 @@ class Regrtest: self.test_count_width = len(self.test_count) - 1 if self.ns.use_mp: - self._run_tests_mp() + from test.libregrtest.runtest_mp import run_tests_multiprocess + run_tests_multiprocess(self) else: self._run_tests_sequential() @@ -518,7 +429,8 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") if self.ns.slaveargs is not None: - slave_runner(self.ns.slaveargs) + from test.libregrtest.runtest_mp import run_tests_slave + run_tests_slave(self.ns.slaveargs) self.find_tests(tests) self.run_tests() self.display_result() @@ -526,26 +438,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -# We do not use a generator so multiple threads can call next(). -class MultiprocessTests(object): - - """A thread-safe iterator over tests for multiprocess mode.""" - - def __init__(self, tests): - self.interrupted = False - self.lock = threading.Lock() - self.tests = tests - - def __iter__(self): - return self - - def __next__(self): - with self.lock: - if self.interrupted: - raise StopIteration('tests interrupted') - return next(self.tests) - - def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index d8c0eb2..60b8f5f 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -1,7 +1,6 @@ import faulthandler import importlib import io -import json import os import sys import time @@ -22,38 +21,6 @@ INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process -def run_test_in_subprocess(testname, ns): - """Run the given test in a subprocess with --slaveargs. - - ns is the option Namespace parsed from command-line arguments. regrtest - is invoked in a subprocess with the --slaveargs argument; when the - subprocess exits, its return code, stdout and stderr are returned as a - 3-tuple. - """ - from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) - # Running the child from the same working directory as regrtest's original - # invocation ensures that TEMPDIR for the child is the same when - # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], - stdout=PIPE, stderr=PIPE, - universal_newlines=True, - close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() - return retcode, stdout, stderr - - # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py new file mode 100644 index 0000000..b55c11a --- /dev/null +++ b/Lib/test/libregrtest/runtest_mp.py @@ -0,0 +1,158 @@ +import json +import os +import re +import sys +import traceback +import unittest +from queue import Queue +from test import support +try: + import threading +except ImportError: + print("Multiprocess option requires thread support") + sys.exit(2) + +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR + + +debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") + + +def run_tests_in_subprocess(testname, ns): + """Run the given test in a subprocess with --slaveargs. + + ns is the option Namespace parsed from command-line arguments. regrtest + is invoked in a subprocess with the --slaveargs argument; when the + subprocess exits, its return code, stdout and stderr are returned as a + 3-tuple. + """ + from subprocess import Popen, PIPE + base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + + ['-X', 'faulthandler', '-m', 'test.regrtest']) + + slaveargs = ( + (testname, ns.verbose, ns.quiet), + dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, failfast=ns.failfast, + match_tests=ns.match_tests)) + # Running the child from the same working directory as regrtest's original + # invocation ensures that TEMPDIR for the child is the same when + # sysconfig.is_python_build() is true. See issue 15300. + popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + stdout=PIPE, stderr=PIPE, + universal_newlines=True, + close_fds=(os.name != 'nt'), + cwd=support.SAVEDCWD) + stdout, stderr = popen.communicate() + retcode = popen.wait() + return retcode, stdout, stderr + + +def run_tests_slave(slaveargs): + args, kwargs = json.loads(slaveargs) + if kwargs.get('huntrleaks'): + unittest.BaseTestSuite._cleanup = False + try: + result = runtest(*args, **kwargs) + except KeyboardInterrupt: + result = INTERRUPTED, '' + except BaseException as e: + traceback.print_exc() + result = CHILD_ERROR, str(e) + sys.stdout.flush() + print() # Force a newline (just in case) + print(json.dumps(result)) + sys.exit(0) + + +# We do not use a generator so multiple threads can call next(). +class MultiprocessIterator: + + """A thread-safe iterator over tests for multiprocess mode.""" + + def __init__(self, tests): + self.interrupted = False + self.lock = threading.Lock() + self.tests = tests + + def __iter__(self): + return self + + def __next__(self): + with self.lock: + if self.interrupted: + raise StopIteration('tests interrupted') + return next(self.tests) + + +class MultiprocessThread(threading.Thread): + def __init__(self, pending, output, ns): + super().__init__() + self.pending = pending + self.output = output + self.ns = ns + + def run(self): + # A worker thread. + try: + while True: + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return + retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + # Strip last refcount output line if it exists, since it + # comes from the shutdown of the interpreter in the subcommand. + stderr = debug_output_pat.sub("", stderr) + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return + if not result: + self.output.put((None, None, None, None)) + return + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + except BaseException: + self.output.put((None, None, None, None)) + raise + + +def run_tests_multiprocess(regrtest): + output = Queue() + pending = MultiprocessIterator(regrtest.tests) + + workers = [MultiprocessThread(pending, output, regrtest.ns) + for i in range(regrtest.ns.use_mp)] + for worker in workers: + worker.start() + finished = 0 + test_index = 1 + try: + while finished < regrtest.ns.use_mp: + test, stdout, stderr, result = output.get() + if test is None: + finished += 1 + continue + regrtest.accumulate_result(test, result) + regrtest.display_progress(test_index, test) + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + sys.stdout.flush() + sys.stderr.flush() + if result[0] == INTERRUPTED: + raise KeyboardInterrupt + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) + test_index += 1 + except KeyboardInterrupt: + regrtest.interrupted = True + pending.interrupted = True + for worker in workers: + worker.join() -- cgit v0.12 From bd1a72c455a18e73e59c502cb28cb0d0667e1087 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:36:27 +0200 Subject: Issue #25220: Enhance regrtest --coverage Add a new Regrtest.run_test() method to ensure that --coverage pass the same options to the runtest() function. --- Lib/test/libregrtest/main.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b66f045..41c32d6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -346,7 +346,18 @@ class Regrtest: print(count(len(self.bad), 'test'), "failed again:") printlist(self.bad) - def _run_tests_sequential(self): + def run_test(self, test): + result = runtest(test, + self.ns.verbose, + self.ns.quiet, + self.ns.huntrleaks, + output_on_failure=self.ns.verbose3, + timeout=self.ns.timeout, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) + self.accumulate_result(test, result) + + def run_tests_sequential(self): save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): @@ -354,19 +365,15 @@ class Regrtest: if self.ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'runtest(test, self.ns.verbose, self.ns.quiet, timeout=self.ns.timeout)' + cmd = 'self.run_test(test)' self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - result = runtest(test, self.ns.verbose, self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) + self.run_test(test) except KeyboardInterrupt: self.interrupted = True break + if self.ns.findleaks: gc.collect() if gc.garbage: @@ -376,13 +383,14 @@ class Regrtest: # them again self.found_garbage.extend(gc.garbage) del gc.garbage[:] + # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + support.verbose = self.ns.verbose # Tell tests to be moderately quiet support.use_resources = self.ns.use_resources if self.ns.forever: @@ -404,7 +412,7 @@ class Regrtest: from test.libregrtest.runtest_mp import run_tests_multiprocess run_tests_multiprocess(self) else: - self._run_tests_sequential() + self.run_tests_sequential() def finalize(self): if self.next_single_filename: -- cgit v0.12 From 37554525aab167c88e61cac0d78dfdefc31fdee9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:37:14 +0200 Subject: Issue #25220: regrtest setups Python after parsing command line options --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 41c32d6..2716536 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -431,8 +431,8 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - setup_python() self.ns = _parse_args(sys.argv[1:], **kwargs) + setup_python() self.setup_regrtest() if self.ns.wait: input("Press any key to continue...") -- cgit v0.12 From 6448b8041a80d29de47799cb8eccd94fd1dde3be Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:43:33 +0200 Subject: Issue #25220: truncate some long lines in libregrtest/*.py --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++------- Lib/test/libregrtest/runtest_mp.py | 12 ++++++++---- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2716536..ee1591d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -63,7 +63,8 @@ def setup_python(): # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) for path in module.__path__] + module.__path__ = [os.path.abspath(path) + for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) @@ -159,8 +160,8 @@ class Regrtest: if self.ns.quiet: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" - print(fmt.format( - self.test_count_width, test_index, self.test_count, len(self.bad), test)) + print(fmt.format(self.test_count_width, test_index, + self.test_count, len(self.bad), test)) sys.stdout.flush() def setup_regrtest(self): @@ -252,7 +253,10 @@ class Regrtest: self.ns.args = [] # For a partial run, we do not need to clutter the output. - if self.ns.verbose or self.ns.header or not (self.ns.quiet or self.ns.single or self.tests or self.ns.args): + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), @@ -283,7 +287,8 @@ class Regrtest: try: del self.selected[:self.selected.index(self.ns.start)] except ValueError: - print("Couldn't find starting test (%s), using all tests" % self.ns.start) + print("Couldn't find starting test (%s), using all tests" + % self.ns.start) if self.ns.randomize: if self.ns.random_seed is None: @@ -297,12 +302,16 @@ class Regrtest: # print a newline after ^C print() print("Test suite interrupted by signal SIGINT.") - omitted = set(self.selected) - set(self.good) - set(self.bad) - set(self.skipped) + executed = set(self.good) | set(self.bad) | set(self.skipped) + omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: - if not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1: + if (not self.bad + and not self.skipped + and not self.interrupted + and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b55c11a..c5a6b3d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -103,20 +103,23 @@ class MultiprocessThread(threading.Thread): except StopIteration: self.output.put((None, None, None, None)) return - retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) + retcode, stdout, stderr = run_tests_in_subprocess(test, + self.ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) return if not result: self.output.put((None, None, None, None)) return result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) except BaseException: self.output.put((None, None, None, None)) raise @@ -149,7 +152,8 @@ def run_tests_multiprocess(regrtest): if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: - raise Exception("Child error on {}: {}".format(test, result[1])) + msg = "Child error on {}: {}".format(test, result[1]) + raise Exception(msg) test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True -- cgit v0.12 From 86e8c31b8da163eb1f412c68ef96c2a4fda7adcf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:50:19 +0200 Subject: Issue #25220, libregrtest: Remove unused import --- Lib/test/libregrtest/main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ee1591d..306beb8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,10 +19,6 @@ try: import gc except ImportError: gc = None -try: - import threading -except ImportError: - threading = None # When tests are run from the Python build directory, it is best practice -- cgit v0.12 From 02319804ea4b92e11fef6adce63698da8c44052a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Sep 2015 23:52:33 +0200 Subject: Don't strip refcount in libregrtest/runtest_mp.py Python doesn't display the refcount anymore by default. It only displays it when -X showrefcount command line option is used, which is not the case here. regrtest can be run with -X showrefcount, the option is not inherited by child processes. --- Lib/test/libregrtest/runtest_mp.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index c5a6b3d..8311ecf 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -15,9 +15,6 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") - - def run_tests_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -105,9 +102,6 @@ class MultiprocessThread(threading.Thread): return retcode, stdout, stderr = run_tests_in_subprocess(test, self.ns) - # Strip last refcount output line if it exists, since it - # comes from the shutdown of the interpreter in the subcommand. - stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) -- cgit v0.12 From 76f756d9344157d893ce49503327af315c7c3b9c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:33:29 +0200 Subject: Issue #25220: Enhance regrtest -jN Running the Python test suite with -jN now: - Display the duration of tests which took longer than 30 seconds - Display the tests currently running since at least 30 seconds - Display the tests we are waiting for when the test suite is interrupted Clenaup also run_test_in_subprocess() code. --- Lib/test/libregrtest/runtest_mp.py | 127 ++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 37 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8311ecf..71096d3 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,7 +1,7 @@ import json import os -import re import sys +import time import traceback import unittest from queue import Queue @@ -15,7 +15,12 @@ except ImportError: from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR -def run_tests_in_subprocess(testname, ns): +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + +def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. ns is the option Namespace parsed from command-line arguments. regrtest @@ -24,26 +29,33 @@ def run_tests_in_subprocess(testname, ns): 3-tuple. """ from subprocess import Popen, PIPE - base_cmd = ([sys.executable] + support.args_from_interpreter_flags() + - ['-X', 'faulthandler', '-m', 'test.regrtest']) - - slaveargs = ( - (testname, ns.verbose, ns.quiet), - dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, failfast=ns.failfast, - match_tests=ns.match_tests)) + + args = (testname, ns.verbose, ns.quiet) + kwargs = dict(huntrleaks=ns.huntrleaks, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + timeout=ns.timeout, + failfast=ns.failfast, + match_tests=ns.match_tests) + slaveargs = (args, kwargs) + slaveargs = json.dumps(slaveargs) + + cmd = [sys.executable, *support.args_from_interpreter_flags(), + '-X', 'faulthandler', + '-m', 'test.regrtest', + '--slaveargs', slaveargs] + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. - popen = Popen(base_cmd + ['--slaveargs', json.dumps(slaveargs)], + popen = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), cwd=support.SAVEDCWD) - stdout, stderr = popen.communicate() - retcode = popen.wait() + with popen: + stdout, stderr = popen.communicate() + retcode = popen.wait() return retcode, stdout, stderr @@ -90,30 +102,45 @@ class MultiprocessThread(threading.Thread): self.pending = pending self.output = output self.ns = ns + self.current_test = None + self.start_time = None + + def _runtest(self): + try: + test = next(self.pending) + except StopIteration: + self.output.put((None, None, None, None)) + return True + + try: + self.start_time = time.monotonic() + self.current_test = test + + retcode, stdout, stderr = run_test_in_subprocess(test, self.ns) + finally: + self.current_test = None + + stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return True + + if not result: + self.output.put((None, None, None, None)) + return True + + result = json.loads(result) + self.output.put((test, stdout.rstrip(), stderr.rstrip(), + result)) + return False def run(self): - # A worker thread. try: - while True: - try: - test = next(self.pending) - except StopIteration: - self.output.put((None, None, None, None)) - return - retcode, stdout, stderr = run_tests_in_subprocess(test, - self.ns) - stdout, _, result = stdout.strip().rpartition("\n") - if retcode != 0: - result = (CHILD_ERROR, "Exit code %s" % retcode) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) - return - if not result: - self.output.put((None, None, None, None)) - return - result = json.loads(result) - self.output.put((test, stdout.rstrip(), stderr.rstrip(), - result)) + stop = False + while not stop: + stop = self._runtest() except BaseException: self.output.put((None, None, None, None)) raise @@ -136,13 +163,33 @@ def run_tests_multiprocess(regrtest): finished += 1 continue regrtest.accumulate_result(test, result) - regrtest.display_progress(test_index, test) + + # Display progress + text = test + ok, test_time = result + if (ok not in (CHILD_ERROR, INTERRUPTED) + and test_time >= PROGRESS_MIN_TIME): + text += ' (%.0f sec)' % test_time + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + if running: + text += ' -- running: %s' % ', '.join(running) + regrtest.display_progress(test_index, text) + + # Copy stdout and stderr from the child process if stdout: print(stdout) if stderr: print(stderr, file=sys.stderr) sys.stdout.flush() sys.stderr.flush() + if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: @@ -152,5 +199,11 @@ def run_tests_multiprocess(regrtest): except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True + print() + + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if running: + print("Waiting for %s" % ', '.join(running)) for worker in workers: worker.join() -- cgit v0.12 From f33536c4308ef1b56a062391ac955c746423e1ee Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:48:27 +0200 Subject: Issue #25220: Use print(flush=True) in libregrtest --- Lib/test/libregrtest/main.py | 7 +++---- Lib/test/libregrtest/refleak.py | 10 ++++------ Lib/test/libregrtest/runtest.py | 14 +++++--------- Lib/test/libregrtest/runtest_mp.py | 9 +++------ 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 306beb8..7440e9d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -157,8 +157,8 @@ class Regrtest: return fmt = "[{1:{0}}{2}/{3}] {4}" if self.bad else "[{1:{0}}{2}] {4}" print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test)) - sys.stdout.flush() + self.test_count, len(self.bad), test), + flush=True) def setup_regrtest(self): if self.ns.huntrleaks: @@ -333,8 +333,7 @@ class Regrtest: if self.ns.verbose2 and self.bad: print("Re-running failed tests in verbose mode") for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test) - sys.stdout.flush() + print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index dd0d05d..db8a445 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -44,13 +44,12 @@ def dash_R(the_module, test, indirect_test, huntrleaks): alloc_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) - print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr) - sys.stderr.flush() + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) - sys.stderr.write('.') - sys.stderr.flush() + print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before @@ -74,8 +73,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) - print(msg, file=sys.stderr) - sys.stderr.flush() + print(msg, file=sys.stderr, flush=True) with open(fname, "a") as refrep: print(msg, file=refrep) refrep.flush() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 60b8f5f..f57784d 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -161,27 +161,23 @@ def runtest_inner(test, verbose, quiet, test_time = time.time() - start_time except support.ResourceDenied as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return RESOURCE_DENIED, test_time except unittest.SkipTest as msg: if not quiet: - print(test, "skipped --", msg) - sys.stdout.flush() + print(test, "skipped --", msg, flush=True) return SKIPPED, test_time except KeyboardInterrupt: raise except support.TestFailed as msg: if display_failure: - print("test", test, "failed --", msg, file=sys.stderr) + print("test", test, "failed --", msg, file=sys.stderr, flush=True) else: - print("test", test, "failed", file=sys.stderr) - sys.stderr.flush() + print("test", test, "failed", file=sys.stderr, flush=True) return FAILED, test_time except: msg = traceback.format_exc() - print("test", test, "crashed --", msg, file=sys.stderr) - sys.stderr.flush() + print("test", test, "crashed --", msg, file=sys.stderr, flush=True) return FAILED, test_time else: if refleak: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 71096d3..1a82b3d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -70,9 +70,8 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) - sys.stdout.flush() print() # Force a newline (just in case) - print(json.dumps(result)) + print(json.dumps(result), flush=True) sys.exit(0) @@ -184,11 +183,9 @@ def run_tests_multiprocess(regrtest): # Copy stdout and stderr from the child process if stdout: - print(stdout) + print(stdout, flush=True) if stderr: - print(stderr, file=sys.stderr) - sys.stdout.flush() - sys.stderr.flush() + print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt -- cgit v0.12 From c7eab0528c00791da8368abe950a7b6f97aa747d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 00:59:35 +0200 Subject: Issue #25220, libregrtest: Cleanup setup code --- Lib/test/libregrtest/main.py | 96 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 7440e9d..f410bfd 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -31,7 +31,7 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(): +def setup_python(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -80,6 +80,38 @@ def setup_python(): newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + if gc is not None: + gc.set_threshold(ns.threshold) + else: + print('No GC available, ignore --threshold.') + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + class Regrtest: """Execute a test suite. @@ -161,36 +193,6 @@ class Regrtest: flush=True) def setup_regrtest(self): - if self.ns.huntrleaks: - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if self.ns.memlimit is not None: - support.set_memlimit(self.ns.memlimit) - - if self.ns.threshold is not None: - if gc is not None: - gc.set_threshold(self.ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if self.ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - if self.ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not @@ -202,19 +204,9 @@ class Regrtest: print('No GC available, disabling --findleaks') self.ns.findleaks = False - if self.ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - # Strip .py extensions. removepy(self.ns.args) - if self.ns.trace: - import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) - def find_tests(self, tests): self.tests = tests @@ -278,7 +270,7 @@ class Regrtest: except IndexError: pass - # Remove all the self.selected tests that precede start if it's set. + # Remove all the selected tests that precede start if it's set. if self.ns.start: try: del self.selected[:self.selected.index(self.ns.start)] @@ -362,11 +354,18 @@ class Regrtest: self.accumulate_result(test, result) def run_tests_sequential(self): + if self.ns.trace: + import trace + self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, + sys.base_exec_prefix, + tempfile.gettempdir()], + trace=False, count=True) + save_modules = sys.modules.keys() for test_index, test in enumerate(self.tests, 1): self.display_progress(test_index, test) - if self.ns.trace: + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = 'self.run_test(test)' @@ -426,7 +425,7 @@ class Regrtest: else: os.unlink(self.next_single_filename) - if self.ns.trace: + if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) @@ -436,15 +435,18 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = _parse_args(sys.argv[1:], **kwargs) - setup_python() + setup_python(self.ns) self.setup_regrtest() - if self.ns.wait: - input("Press any key to continue...") + if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: + input("Press any key to continue...") + self.find_tests(tests) self.run_tests() + self.display_result() self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) -- cgit v0.12 From 234cbef39fe7f154d46afcc6fe865db2e120bba2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:13:53 +0200 Subject: Issue #25220, libregrtest: Move setup_python() to a new submodule --- Lib/test/libregrtest/main.py | 125 +++++------------------------------------- Lib/test/libregrtest/setup.py | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 110 deletions(-) create mode 100644 Lib/test/libregrtest/setup.py diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index f410bfd..fbbfa73 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,19 +1,16 @@ -import faulthandler import os import platform import random import re -import signal import sys import sysconfig import tempfile import textwrap -import unittest from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.refleak import warm_caches from test.libregrtest.cmdline import _parse_args +from test.libregrtest.setup import setup_python from test import support try: import gc @@ -31,88 +28,6 @@ else: TEMPDIR = os.path.abspath(TEMPDIR) -def setup_python(ns): - # Display the Python traceback on fatal errors (e.g. segfault) - faulthandler.enable(all_threads=True) - - # Display the Python traceback on SIGALRM or SIGUSR1 signal - signals = [] - if hasattr(signal, 'SIGALRM'): - signals.append(signal.SIGALRM) - if hasattr(signal, 'SIGUSR1'): - signals.append(signal.SIGUSR1) - for signum in signals: - faulthandler.register(signum, chain=True) - - replace_stdout() - support.record_original_stdout(sys.stdout) - - # Some times __path__ and __file__ are not absolute (e.g. while running from - # Lib/) and, if we change the CWD to run the tests in a temporary dir, some - # imports might fail. This affects only the modules imported before os.chdir(). - # These modules are searched first in sys.path[0] (so '' -- the CWD) and if - # they are found in the CWD their __file__ and __path__ will be relative (this - # happens before the chdir). All the modules imported after the chdir, are - # not found in the CWD, and since the other paths in sys.path[1:] are absolute - # (site.py absolutize them), the __file__ and __path__ will be absolute too. - # Therefore it is necessary to absolutize manually the __file__ and __path__ of - # the packages to prevent later imports to fail when the CWD is different. - for module in sys.modules.values(): - if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] - if hasattr(module, '__file__'): - module.__file__ = os.path.abspath(module.__file__) - - # MacOSX (a.k.a. Darwin) has a default stack size that is too small - # for deeply recursive regular expressions. We see this as crashes in - # the Python test suite when running test_re.py and test_sre.py. The - # fix is to set the stack limit to 2048. - # This approach may also be useful for other Unixy platforms that - # suffer from small default stack limits. - if sys.platform == 'darwin': - try: - import resource - except ImportError: - pass - else: - soft, hard = resource.getrlimit(resource.RLIMIT_STACK) - newsoft = min(hard, max(soft, 1024*2048)) - resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) - - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False - - # Avoid false positives due to various caches - # filling slowly with random data: - warm_caches() - - if ns.memlimit is not None: - support.set_memlimit(ns.memlimit) - - if ns.threshold is not None: - if gc is not None: - gc.set_threshold(ns.threshold) - else: - print('No GC available, ignore --threshold.') - - if ns.nowindows: - import msvcrt - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass - else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - - class Regrtest: """Execute a test suite. @@ -192,8 +107,14 @@ class Regrtest: self.test_count, len(self.bad), test), flush=True) - def setup_regrtest(self): - if self.ns.findleaks: + def parse_args(self, kwargs): + ns = _parse_args(sys.argv[1:], **kwargs) + + if ns.threshold is not None and gc is None: + print('No GC available, ignore --threshold.') + ns.threshold = None + + if ns.findleaks: if gc is not None: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only @@ -202,10 +123,12 @@ class Regrtest: #gc.set_debug(gc.DEBUG_SAVEALL) else: print('No GC available, disabling --findleaks') - self.ns.findleaks = False + ns.findleaks = False # Strip .py extensions. - removepy(self.ns.args) + removepy(ns.args) + + return ns def find_tests(self, tests): self.tests = tests @@ -434,9 +357,9 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): - self.ns = _parse_args(sys.argv[1:], **kwargs) + self.ns = self.parse_args(kwargs) + setup_python(self.ns) - self.setup_regrtest() if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave @@ -452,24 +375,6 @@ class Regrtest: sys.exit(len(self.bad) > 0 or self.interrupted) -def replace_stdout(): - """Set stdout encoder error handler to backslashreplace (as stderr error - handler) to avoid UnicodeEncodeError when printing a traceback""" - import atexit - - stdout = sys.stdout - sys.stdout = open(stdout.fileno(), 'w', - encoding=stdout.encoding, - errors="backslashreplace", - closefd=False, - newline='\n') - - def restore_stdout(): - sys.stdout.close() - sys.stdout = stdout - atexit.register(restore_stdout) - - def removepy(names): if not names: return diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py new file mode 100644 index 0000000..a7dfa79 --- /dev/null +++ b/Lib/test/libregrtest/setup.py @@ -0,0 +1,108 @@ +import atexit +import faulthandler +import os +import signal +import sys +import unittest +from test import support +try: + import gc +except ImportError: + gc = None + +from test.libregrtest.refleak import warm_caches + + +def setup_python(ns): + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + + replace_stdout() + support.record_original_stdout(sys.stdout) + + # Some times __path__ and __file__ are not absolute (e.g. while running from + # Lib/) and, if we change the CWD to run the tests in a temporary dir, some + # imports might fail. This affects only the modules imported before os.chdir(). + # These modules are searched first in sys.path[0] (so '' -- the CWD) and if + # they are found in the CWD their __file__ and __path__ will be relative (this + # happens before the chdir). All the modules imported after the chdir, are + # not found in the CWD, and since the other paths in sys.path[1:] are absolute + # (site.py absolutize them), the __file__ and __path__ will be absolute too. + # Therefore it is necessary to absolutize manually the __file__ and __path__ of + # the packages to prevent later imports to fail when the CWD is different. + for module in sys.modules.values(): + if hasattr(module, '__path__'): + module.__path__ = [os.path.abspath(path) + for path in module.__path__] + if hasattr(module, '__file__'): + module.__file__ = os.path.abspath(module.__file__) + + # MacOSX (a.k.a. Darwin) has a default stack size that is too small + # for deeply recursive regular expressions. We see this as crashes in + # the Python test suite when running test_re.py and test_sre.py. The + # fix is to set the stack limit to 2048. + # This approach may also be useful for other Unixy platforms that + # suffer from small default stack limits. + if sys.platform == 'darwin': + try: + import resource + except ImportError: + pass + else: + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + newsoft = min(hard, max(soft, 1024*2048)) + resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) + + if ns.huntrleaks: + unittest.BaseTestSuite._cleanup = False + + # Avoid false positives due to various caches + # filling slowly with random data: + warm_caches() + + if ns.memlimit is not None: + support.set_memlimit(ns.memlimit) + + if ns.threshold is not None: + gc.set_threshold(ns.threshold) + + if ns.nowindows: + import msvcrt + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + + +def replace_stdout(): + """Set stdout encoder error handler to backslashreplace (as stderr error + handler) to avoid UnicodeEncodeError when printing a traceback""" + stdout = sys.stdout + sys.stdout = open(stdout.fileno(), 'w', + encoding=stdout.encoding, + errors="backslashreplace", + closefd=False, + newline='\n') + + def restore_stdout(): + sys.stdout.close() + sys.stdout = stdout + atexit.register(restore_stdout) -- cgit v0.12 From 8bb19f094ba6ba12f70a6458729c911d93c85209 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:32:39 +0200 Subject: Issue #25220, libregrtest: Add runtest_ns() function * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/main.py | 17 ++++++----------- Lib/test/libregrtest/runtest.py | 7 +++++++ Lib/test/libregrtest/runtest_mp.py | 24 +++++++++++++----------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fbbfa73..df2329f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest, + findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_python @@ -251,8 +251,7 @@ class Regrtest: print("Re-running test %r in verbose mode" % test, flush=True) try: self.ns.verbose = True - ok = runtest(test, True, self.ns.quiet, self.ns.huntrleaks, - timeout=self.ns.timeout) + ok = runtest_ns(test, True, self.ns) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -266,14 +265,10 @@ class Regrtest: printlist(self.bad) def run_test(self, test): - result = runtest(test, - self.ns.verbose, - self.ns.quiet, - self.ns.huntrleaks, - output_on_failure=self.ns.verbose3, - timeout=self.ns.timeout, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) + result = runtest_ns(test, self.ns.verbose, self.ns, + output_on_failure=self.ns.verbose3, + failfast=self.ns.failfast, + match_tests=self.ns.match_tests) self.accumulate_result(test, result) def run_tests_sequential(self): diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index f57784d..fb7f821 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,6 +53,13 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) +def runtest_ns(test, verbose, ns, **kw): + return runtest(test, verbose, ns.quiet, + huntrleaks=ns.huntrleaks, + timeout=ns.timeout, + **kw) + + def runtest(test, verbose, quiet, huntrleaks=False, use_resources=None, output_on_failure=False, failfast=False, match_tests=None, diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 1a82b3d..74424c1 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import os import sys import time import traceback +import types import unittest from queue import Queue from test import support @@ -30,14 +31,8 @@ def run_test_in_subprocess(testname, ns): """ from subprocess import Popen, PIPE - args = (testname, ns.verbose, ns.quiet) - kwargs = dict(huntrleaks=ns.huntrleaks, - use_resources=ns.use_resources, - output_on_failure=ns.verbose3, - timeout=ns.timeout, - failfast=ns.failfast, - match_tests=ns.match_tests) - slaveargs = (args, kwargs) + ns_dict = vars(ns) + slaveargs = (ns_dict, testname) slaveargs = json.dumps(slaveargs) cmd = [sys.executable, *support.args_from_interpreter_flags(), @@ -60,11 +55,18 @@ def run_test_in_subprocess(testname, ns): def run_tests_slave(slaveargs): - args, kwargs = json.loads(slaveargs) - if kwargs.get('huntrleaks'): + ns_dict, testname = json.loads(slaveargs) + ns = types.SimpleNamespace(**ns_dict) + + if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False + try: - result = runtest(*args, **kwargs) + result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + use_resources=ns.use_resources, + output_on_failure=ns.verbose3, + failfast=ns.failfast, + match_tests=ns.match_tests) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v0.12 From ecef622fec197f4377b5b5ec58f80f5fd000210f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 01:39:28 +0200 Subject: Issue #25220, libregrtest: Call setup_python(ns) in the slaves Slaves (child processes running tests for regrtest -jN) now inherit --memlimit/-M, --threshold/-t and --nowindows/-n options. * -M, -t and -n are now supported with -jN * Factorize code to run tests. * run_test_in_subprocess() now pass the whole "ns" namespace to the child process. --- Lib/test/libregrtest/cmdline.py | 2 -- Lib/test/libregrtest/main.py | 5 +++-- Lib/test/libregrtest/runtest_mp.py | 8 ++++---- Lib/test/test_regrtest.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 19c52a2..e55e53f 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -295,8 +295,6 @@ def _parse_args(args, **kwargs): parser.error("-T and -j don't go together!") if ns.use_mp and ns.findleaks: parser.error("-l and -j don't go together!") - if ns.use_mp and ns.memlimit: - parser.error("-M and -j don't go together!") if ns.failfast and not (ns.verbose or ns.verbose3): parser.error("-G/--failfast needs either -v or -W") diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index df2329f..e685970 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -354,14 +354,15 @@ class Regrtest: def main(self, tests=None, **kwargs): self.ns = self.parse_args(kwargs) - setup_python(self.ns) - if self.ns.slaveargs is not None: from test.libregrtest.runtest_mp import run_tests_slave run_tests_slave(self.ns.slaveargs) + if self.ns.wait: input("Press any key to continue...") + setup_python(self.ns) + self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 74424c1..47393aa 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,8 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.setup import setup_python # Minimum duration of a test to display its duration or to mention that @@ -58,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - if ns.huntrleaks: - unittest.BaseTestSuite._cleanup = False + setup_python(ns) try: - result = runtest_ns(testname, ns.verbose, ns.quiet, ns, + result = runtest_ns(testname, ns.verbose, ns, use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 54bbfe4..c277e10 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -214,7 +214,6 @@ class ParseArgsTestCase(unittest.TestCase): self.checkError([opt, 'foo'], 'invalid int value') self.checkError([opt, '2', '-T'], "don't go together") self.checkError([opt, '2', '-l'], "don't go together") - self.checkError([opt, '2', '-M', '4G'], "don't go together") def test_coverage(self): for opt in '-T', '--coverage': -- cgit v0.12 From 00b8f9bb9ea1a246acc27f371c123016d50842aa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:02:49 +0200 Subject: Issue #25274: Workaround test_sys crash just to keep buildbots usable --- Lib/test/test_sys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 83549bc..ccbf262 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -208,7 +208,10 @@ class SysModuleTest(unittest.TestCase): def f(): f() try: - for i in (50, 1000): + # FIXME: workaround crash for the issue #25274 + # FIXME: until the crash is fixed + #for i in (50, 1000): + for i in (150, 1000): # Issue #5392: stack overflow after hitting recursion limit twice sys.setrecursionlimit(i) self.assertRaises(RecursionError, f) -- cgit v0.12 From a204502dbf38318ea926eeccab886dcb52305a16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:17:28 +0200 Subject: Issue #25220, libregrtest: Set support.use_resources in setup_tests() * Rename setup_python() to setup_tests() * Remove use_resources parameter of runtest() --- Lib/test/libregrtest/main.py | 5 ++--- Lib/test/libregrtest/runtest.py | 5 +---- Lib/test/libregrtest/runtest_mp.py | 5 ++--- Lib/test/libregrtest/setup.py | 4 +++- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e685970..9ca407a 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -10,7 +10,7 @@ from test.libregrtest.runtest import ( findtests, runtest_ns, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests from test import support try: import gc @@ -312,7 +312,6 @@ class Regrtest: def run_tests(self): support.verbose = self.ns.verbose # Tell tests to be moderately quiet - support.use_resources = self.ns.use_resources if self.ns.forever: def test_forever(tests): @@ -361,7 +360,7 @@ class Regrtest: if self.ns.wait: input("Press any key to continue...") - setup_python(self.ns) + setup_tests(self.ns) self.find_tests(tests) self.run_tests() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index fb7f821..a3d4e79 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -61,7 +61,7 @@ def runtest_ns(test, verbose, ns, **kw): def runtest(test, verbose, quiet, - huntrleaks=False, use_resources=None, + huntrleaks=False, output_on_failure=False, failfast=False, match_tests=None, timeout=None): """Run a single test. @@ -71,7 +71,6 @@ def runtest(test, verbose, quiet, quiet -- if true, don't print 'skipped' messages (probably redundant) huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments - use_resources -- list of extra resources to use output_on_failure -- if true, display test output on failure timeout -- dump the traceback and exit if a test takes more than timeout seconds @@ -86,8 +85,6 @@ def runtest(test, verbose, quiet, PASSED test passed """ - if use_resources is not None: - support.use_resources = use_resources use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 47393aa..8332a0b 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -14,7 +14,7 @@ except ImportError: sys.exit(2) from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR -from test.libregrtest.setup import setup_python +from test.libregrtest.setup import setup_tests # Minimum duration of a test to display its duration or to mention that @@ -59,11 +59,10 @@ def run_tests_slave(slaveargs): ns_dict, testname = json.loads(slaveargs) ns = types.SimpleNamespace(**ns_dict) - setup_python(ns) + setup_tests(ns) try: result = runtest_ns(testname, ns.verbose, ns, - use_resources=ns.use_resources, output_on_failure=ns.verbose3, failfast=ns.failfast, match_tests=ns.match_tests) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index a7dfa79..6a1c308 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -13,7 +13,7 @@ except ImportError: from test.libregrtest.refleak import warm_caches -def setup_python(ns): +def setup_tests(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) @@ -91,6 +91,8 @@ def setup_python(ns): msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + support.use_resources = ns.use_resources + def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error -- cgit v0.12 From 6f20a2e01ffe897c20d0085f63371c467d118dac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:32:11 +0200 Subject: Issue #25220, libregrtest: Pass directly ns to runtest() * Remove runtest_ns(): pass directly ns to runtest(). * Create also Regrtest.rerun_failed_tests() method. * Inline again Regrtest.run_test(): it's no more justified to have a method --- Lib/test/libregrtest/main.py | 62 ++++++++++++++++++++------------------ Lib/test/libregrtest/runtest.py | 20 ++++++------ Lib/test/libregrtest/runtest_mp.py | 7 ++--- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9ca407a..c102ee0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,7 +7,7 @@ import sysconfig import tempfile import textwrap from test.libregrtest.runtest import ( - findtests, runtest_ns, + findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) from test.libregrtest.cmdline import _parse_args from test.libregrtest.setup import setup_tests @@ -208,6 +208,30 @@ class Regrtest: print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def rerun_failed_tests(self): + self.ns.verbose = True + self.ns.failfast = False + self.ns.verbose3 = False + self.ns.match_tests = None + + print("Re-running failed tests in verbose mode") + for test in self.bad[:]: + print("Re-running test %r in verbose mode" % test, flush=True) + try: + self.ns.verbose = True + ok = runtest(self.ns, test) + except KeyboardInterrupt: + # print a newline separate from the ^C + print() + break + else: + if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: + self.bad.remove(test) + else: + if self.bad: + print(count(len(self.bad), 'test'), "failed again:") + printlist(self.bad) + def display_result(self): if self.interrupted: # print a newline after ^C @@ -245,32 +269,6 @@ class Regrtest: print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) - if self.ns.verbose2 and self.bad: - print("Re-running failed tests in verbose mode") - for test in self.bad[:]: - print("Re-running test %r in verbose mode" % test, flush=True) - try: - self.ns.verbose = True - ok = runtest_ns(test, True, self.ns) - except KeyboardInterrupt: - # print a newline separate from the ^C - print() - break - else: - if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: - self.bad.remove(test) - else: - if self.bad: - print(count(len(self.bad), 'test'), "failed again:") - printlist(self.bad) - - def run_test(self, test): - result = runtest_ns(test, self.ns.verbose, self.ns, - output_on_failure=self.ns.verbose3, - failfast=self.ns.failfast, - match_tests=self.ns.match_tests) - self.accumulate_result(test, result) - def run_tests_sequential(self): if self.ns.trace: import trace @@ -286,11 +284,13 @@ class Regrtest: if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - cmd = 'self.run_test(test)' + cmd = ('result = runtest(self.ns, test); ' + 'self.accumulate_result(test, result)') self.tracer.runctx(cmd, globals=globals(), locals=vars()) else: try: - self.run_test(test) + result = runtest(self.ns, test) + self.accumulate_result(test, result) except KeyboardInterrupt: self.interrupted = True break @@ -366,6 +366,10 @@ class Regrtest: self.run_tests() self.display_result() + + if self.ns.verbose2 and self.bad: + self.rerun_failed_tests() + self.finalize() sys.exit(len(self.bad) > 0 or self.interrupted) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index a3d4e79..4cc2588 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -53,17 +53,7 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): return stdtests + sorted(tests) -def runtest_ns(test, verbose, ns, **kw): - return runtest(test, verbose, ns.quiet, - huntrleaks=ns.huntrleaks, - timeout=ns.timeout, - **kw) - - -def runtest(test, verbose, quiet, - huntrleaks=False, - output_on_failure=False, failfast=False, match_tests=None, - timeout=None): +def runtest(ns, test): """Run a single test. test -- the name of the test @@ -85,6 +75,14 @@ def runtest(test, verbose, quiet, PASSED test passed """ + verbose = ns.verbose + quiet = ns.quiet + huntrleaks = ns.huntrleaks + output_on_failure = ns.verbose3 + failfast = ns.failfast + match_tests = ns.match_tests + timeout = ns.timeout + use_timeout = (timeout is not None) if use_timeout: faulthandler.dump_traceback_later(timeout, exit=True) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 8332a0b..df075c1 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,7 +13,7 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest_ns, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR from test.libregrtest.setup import setup_tests @@ -62,10 +62,7 @@ def run_tests_slave(slaveargs): setup_tests(ns) try: - result = runtest_ns(testname, ns.verbose, ns, - output_on_failure=ns.verbose3, - failfast=ns.failfast, - match_tests=ns.match_tests) + result = runtest(ns, testname) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: -- cgit v0.12 From b40843546b1dc558da7217f8c0fe72fbca38e797 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 02:39:22 +0200 Subject: Issue #25220, libregrtest: Cleanup No need to support.verbose in Regrtest.run_tests(), it's always set in runtest(). --- Lib/test/libregrtest/main.py | 17 ++++++++--------- Lib/test/libregrtest/runtest_mp.py | 1 + 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c102ee0..fdb925d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -310,17 +310,16 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) - def run_tests(self): - support.verbose = self.ns.verbose # Tell tests to be moderately quiet + def _test_forever(self, tests): + while True: + for test in tests: + yield test + if self.bad: + return + def run_tests(self): if self.ns.forever: - def test_forever(tests): - while True: - for test in tests: - yield test - if self.bad: - return - self.tests = test_forever(list(self.selected)) + self.tests = _test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index df075c1..6ed8dc2 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -68,6 +68,7 @@ def run_tests_slave(slaveargs): except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) + print() # Force a newline (just in case) print(json.dumps(result), flush=True) sys.exit(0) -- cgit v0.12 From 17f97166766fd2ebe46df0f9fe9308182d141529 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 03:05:43 +0200 Subject: Issue #25220, libregrtest: more verbose output for -jN When the -jN command line option is used, display tests running since at least 30 seconds every minute. --- Lib/test/libregrtest/runtest_mp.py | 39 +++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 6ed8dc2..b31b51e 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,11 +1,11 @@ import json import os +import queue import sys import time import traceback import types import unittest -from queue import Queue from test import support try: import threading @@ -21,6 +21,9 @@ from test.libregrtest.setup import setup_tests # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds +# Display the running tests if nothing happened last N seconds +PROGRESS_UPDATE = 60.0 # seconds + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -145,18 +148,39 @@ class MultiprocessThread(threading.Thread): def run_tests_multiprocess(regrtest): - output = Queue() + output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] for worker in workers: worker.start() + + def get_running(workers): + running = [] + for worker in workers: + current_test = worker.current_test + if not current_test: + continue + dt = time.monotonic() - worker.start_time + if dt >= PROGRESS_MIN_TIME: + running.append('%s (%.0f sec)' % (current_test, dt)) + return running + finished = 0 test_index = 1 + timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: - test, stdout, stderr, result = output.get() + try: + item = output.get(timeout=PROGRESS_UPDATE) + except queue.Empty: + running = get_running(workers) + if running: + print('running: %s' % ', '.join(running)) + continue + + test, stdout, stderr, result = item if test is None: finished += 1 continue @@ -168,14 +192,7 @@ def run_tests_multiprocess(regrtest): if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME): text += ' (%.0f sec)' % test_time - running = [] - for worker in workers: - current_test = worker.current_test - if not current_test: - continue - dt = time.monotonic() - worker.start_time - if dt >= PROGRESS_MIN_TIME: - running.append('%s (%.0f sec)' % (current_test, dt)) + running = get_running(workers) if running: text += ' -- running: %s' % ', '.join(running) regrtest.display_progress(test_index, text) -- cgit v0.12 From 38031143fbab6267fe464a1cb39eeb5c68de38f4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 29 Sep 2015 22:45:05 -0700 Subject: Add an early-out for deque_clear() --- Modules/_collectionsmodule.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 5db7aed..d9df574 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -595,6 +595,9 @@ deque_clear(dequeobject *deque) Py_ssize_t n; PyObject *item; + if (Py_SIZE(deque) == 0) + return; + /* During the process of clearing a deque, decrefs can cause the deque to mutate. To avoid fatal confusion, we have to make the deque empty before clearing the blocks and never refer to -- cgit v0.12 From 9a14214aee22512f833a8d5cf8627ca45627d330 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 13:51:17 +0200 Subject: Issue #25220: Fix "-m test --forever" * Fix "-m test --forever": replace _test_forever() with self._test_forever() * Add unit test for --forever * Add unit test for a failing test * Fix also some pyflakes warnings in libregrtest --- Lib/test/libregrtest/main.py | 2 +- Lib/test/libregrtest/refleak.py | 6 +- Lib/test/libregrtest/runtest_mp.py | 3 +- Lib/test/test_regrtest.py | 133 ++++++++++++++++++++++++++----------- 4 files changed, 100 insertions(+), 44 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fdb925d..e1a99fb 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -319,7 +319,7 @@ class Regrtest: def run_tests(self): if self.ns.forever: - self.tests = _test_forever(list(self.selected)) + self.tests = self._test_forever(list(self.selected)) self.test_count = '' self.test_count_width = 3 else: diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index db8a445..9be0dec 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -46,6 +46,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = 0 for i in range(repcount): indirect_test() alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) @@ -158,6 +160,6 @@ def warm_caches(): for i in range(256): s[i:i+1] # unicode cache - x = [chr(i) for i in range(256)] + [chr(i) for i in range(256)] # int cache - x = list(range(-5, 257)) + list(range(-5, 257)) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index b31b51e..a732d70 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -5,7 +5,6 @@ import sys import time import traceback import types -import unittest from test import support try: import threading @@ -173,7 +172,7 @@ def run_tests_multiprocess(regrtest): try: while finished < regrtest.ns.use_mp: try: - item = output.get(timeout=PROGRESS_UPDATE) + item = output.get(timeout=timeout) except queue.Empty: running = get_running(workers) if running: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c277e10..897598d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -330,33 +330,52 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - parser = re.finditer(r'^\[[0-9]+/[0-9]+\] (%s)$' % self.TESTNAME_REGEX, - output, - re.MULTILINE) - return set(match.group(1) for match in parser) + regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + parser = re.finditer(regex, output, re.MULTILINE) + return list(match.group(1) for match in parser) - def check_executed_tests(self, output, tests, skipped=None): + def check_executed_tests(self, output, tests, skipped=(), failed=(), + randomize=False): if isinstance(tests, str): tests = [tests] - executed = self.parse_executed_tests(output) - self.assertEqual(executed, set(tests), output) + if isinstance(skipped, str): + skipped = [skipped] + if isinstance(failed, str): + failed = [failed] ntest = len(tests) - if skipped: - if isinstance(skipped, str): - skipped = [skipped] - nskipped = len(skipped) - - plural = 's' if nskipped != 1 else '' - names = ' '.join(sorted(skipped)) - expected = (r'%s test%s skipped:\n %s$' - % (nskipped, plural, names)) - self.check_line(output, expected) - - ok = ntest - nskipped - if ok: - self.check_line(output, r'%s test OK\.$' % ok) + nskipped = len(skipped) + nfailed = len(failed) + + executed = self.parse_executed_tests(output) + if randomize: + self.assertEqual(set(executed), set(tests), output) else: - self.check_line(output, r'All %s tests OK\.$' % ntest) + self.assertEqual(executed, tests, output) + + def plural(count): + return 's' if count != 1 else '' + + def list_regex(line_format, tests): + count = len(tests) + names = ' '.join(sorted(tests)) + regex = line_format % (count, plural(count)) + regex = r'%s:\n %s$' % (regex, names) + return regex + + if skipped: + regex = list_regex('%s test%s skipped', skipped) + self.check_line(output, regex) + + if failed: + regex = list_regex('%s test%s failed', failed) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed + if good: + regex = r'%s test%s OK\.$' % (good, plural(good)) + if not skipped and not failed and good > 1: + regex = 'All %s' % regex + self.check_line(output, regex) def parse_random_seed(self, output): match = self.regex_search(r'Using random seed ([0-9]+)', output) @@ -364,24 +383,28 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None): + def run_command(self, args, input=None, exitcode=0): if not input: input = '' - try: - return subprocess.run(args, - check=True, universal_newlines=True, - input=input, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except subprocess.CalledProcessError as exc: - self.fail("%s\n" + proc = subprocess.run(args, + universal_newlines=True, + input=input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + if proc.returncode != exitcode: + self.fail("Command %s failed with exit code %s\n" "\n" "stdout:\n" + "---\n" "%s\n" + "---\n" "\n" "stderr:\n" + "---\n" "%s" - % (str(exc), exc.stdout, exc.stderr)) + "---\n" + % (str(args), proc.returncode, proc.stdout, proc.stderr)) + return proc def run_python(self, args, **kw): @@ -411,11 +434,11 @@ class ProgramsTestCase(BaseTestCase): def check_output(self, output): self.parse_random_seed(output) - self.check_executed_tests(output, self.tests) + self.check_executed_tests(output, self.tests, randomize=True) def run_tests(self, args): - stdout = self.run_python(args) - self.check_output(stdout) + output = self.run_python(args) + self.check_output(output) def test_script_regrtest(self): # Lib/test/regrtest.py @@ -492,8 +515,24 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, input=None): - return self.run_python(['-m', 'test', *args], input=input) + def run_tests(self, *args, **kw): + return self.run_python(['-m', 'test', *args], **kw) + + def test_failing_test(self): + # test a failing test + code = textwrap.dedent(""" + import unittest + + class FailingTest(unittest.TestCase): + def test_failing(self): + self.fail("bug") + """) + test_ok = self.create_test() + test_failing = self.create_test(code=code) + tests = [test_ok, test_failing] + + output = self.run_tests(*tests, exitcode=1) + self.check_executed_tests(output, tests, failed=test_failing) def test_resources(self): # test -u command line option @@ -572,8 +611,7 @@ class ArgsTestCase(BaseTestCase): # test --coverage test = self.create_test() output = self.run_tests("--coverage", test) - executed = self.parse_executed_tests(output) - self.assertEqual(executed, {test}, output) + self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +\(path\)\n' '(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) @@ -584,6 +622,23 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') + def test_forever(self): + # test --forever + code = textwrap.dedent(""" + import unittest + + class ForeverTester(unittest.TestCase): + RUN = 1 + + def test_run(self): + ForeverTester.RUN += 1 + if ForeverTester.RUN > 3: + self.fail("fail at the 3rd runs") + """) + test = self.create_test(code=code) + output = self.run_tests('--forever', test, exitcode=1) + self.check_executed_tests(output, [test]*3, failed=test) + if __name__ == '__main__': unittest.main() -- cgit v0.12 From c51d244fc922610ee58e30dbd99d8d42f32d061b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 22:06:51 +0200 Subject: Issue #25171: Fix compilation issue on OpenBSD in random.c Patch written by Remi Pointel. --- Python/random.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/random.c b/Python/random.c index 8f3e6d6..8caaa95 100644 --- a/Python/random.c +++ b/Python/random.c @@ -364,7 +364,7 @@ _PyOS_URandom(void *buffer, Py_ssize_t size) #ifdef MS_WINDOWS return win32_urandom((unsigned char *)buffer, size, 1); -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) return py_getentropy(buffer, size, 0); #else return dev_urandom_python((char*)buffer, size); @@ -411,7 +411,7 @@ _PyRandom_Init(void) else { #ifdef MS_WINDOWS (void)win32_urandom(secret, secret_size, 0); -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) (void)py_getentropy(secret, secret_size, 1); #else dev_urandom_noraise(secret, secret_size); @@ -427,7 +427,7 @@ _PyRandom_Fini(void) CryptReleaseContext(hCryptProv, 0); hCryptProv = 0; } -#elif PY_GETENTROPY +#elif defined(PY_GETENTROPY) /* nothing to clean */ #else dev_urandom_close(); -- cgit v0.12 From c29f399e7e59c6e3d369568af784958321c151ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Sep 2015 22:50:12 +0200 Subject: Backout change 28d3bcb1bad6: "Try to fix _PyTime_AsTimevalStruct_impl() on OpenBSD", I'm not sure that the change was really needed. I read the test result of an old build because the OpenBSD was 100 builds late. --- Python/pytime.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 53611b1..9889a3b 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs, secs2; + _PyTime_t secs; int us; int res; @@ -467,8 +467,7 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - secs2 = (_PyTime_t)tv->tv_sec; - if (res < 0 || secs2 != secs) { + if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From a53a818c3c67c927bcf8ee118fd29e21045dd9ab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 00:53:09 +0200 Subject: Fix regrtest --coverage on Windows Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of ignored directories. --- Lib/test/libregrtest/main.py | 5 +---- Lib/test/test_regrtest.py | 2 -- Misc/NEWS | 3 +++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1a99fb..c1ce3b1 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -272,10 +272,7 @@ class Regrtest: def run_tests_sequential(self): if self.ns.trace: import trace - self.tracer = trace.Trace(ignoredirs=[sys.base_prefix, - sys.base_exec_prefix, - tempfile.gettempdir()], - trace=False, count=True) + self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 897598d..0f5f22c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -605,8 +605,6 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - @unittest.skipIf(sys.platform == 'win32', - "FIXME: coverage doesn't work on Windows") def test_coverage(self): # test --coverage test = self.create_test() diff --git a/Misc/NEWS b/Misc/NEWS index 72d6fef..a1aa269 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -157,6 +157,9 @@ Documentation Tests ----- +- Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the + list of ignored directories. + - PCbuild\rt.bat now accepts an unlimited number of arguments to pass along to regrtest.py. Previously there was a limit of 9. -- cgit v0.12 From 0d30940dd256da65c91c3bb2d1e135f90eb8c223 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 30 Sep 2015 23:15:02 -0700 Subject: Add fast paths to deque_init() for the common cases --- Modules/_collectionsmodule.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index d9df574..0c64713 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1456,8 +1456,14 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) Py_ssize_t maxlen = -1; char *kwlist[] = {"iterable", "maxlen", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, &iterable, &maxlenobj)) - return -1; + if (kwdargs == NULL) { + if (!PyArg_UnpackTuple(args, "deque()", 0, 2, &iterable, &maxlenobj)) + return -1; + } else { + if (!PyArg_ParseTupleAndKeywords(args, kwdargs, "|OO:deque", kwlist, + &iterable, &maxlenobj)) + return -1; + } if (maxlenobj != NULL && maxlenobj != Py_None) { maxlen = PyLong_AsSsize_t(maxlenobj); if (maxlen == -1 && PyErr_Occurred()) @@ -1468,7 +1474,8 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) } } deque->maxlen = maxlen; - deque_clear(deque); + if (Py_SIZE(deque) > 0) + deque_clear(deque); if (iterable != NULL) { PyObject *rv = deque_extend(deque, iterable); if (rv == NULL) -- cgit v0.12 From b7a8af20ff09efdfa6896ecaff7d036562c17897 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 08:44:03 +0200 Subject: Fix _PyTime_AsTimevalStruct_impl() on OpenBSD On the x86 OpenBSD 5.8 buildbot, the integer overflow check is ignored. Copy the tv_sec variable into a Py_time_t variable instead of "simply" casting it to Py_time_t, to fix the integer overflow check. --- Python/pytime.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/pytime.c b/Python/pytime.c index 9889a3b..53611b1 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -454,7 +454,7 @@ static int _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, _PyTime_round_t round, int raise) { - _PyTime_t secs; + _PyTime_t secs, secs2; int us; int res; @@ -467,7 +467,8 @@ _PyTime_AsTimevalStruct_impl(_PyTime_t t, struct timeval *tv, #endif tv->tv_usec = us; - if (res < 0 || (_PyTime_t)tv->tv_sec != secs) { + secs2 = (_PyTime_t)tv->tv_sec; + if (res < 0 || secs2 != secs) { if (raise) error_time_t_overflow(); return -1; -- cgit v0.12 From 98f223dfa0440529261e921c8b9ec9a955c6d27f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 13:16:43 +0200 Subject: Issue #25277: Set a timeout of 10 minutes in test_eintr using faulthandler to try to debug a hang on the FreeBSD 9 buildbot. Run also eintr_tester.py with python "-u" command line option to try to get the full output on hang/crash. --- Lib/test/eintrdata/eintr_tester.py | 5 +++++ Lib/test/test_eintr.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1e0d91..443ccd5 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -9,6 +9,7 @@ sub-second periodicity (contrarily to signal()). """ import contextlib +import faulthandler import io import os import select @@ -50,6 +51,9 @@ class EINTRBaseTest(unittest.TestCase): signal.setitimer(signal.ITIMER_REAL, cls.signal_delay, cls.signal_period) + # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD + faulthandler.dump_traceback_later(10 * 60, exit=True) + @classmethod def stop_alarm(cls): signal.setitimer(signal.ITIMER_REAL, 0, 0) @@ -58,6 +62,7 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): diff --git a/Lib/test/test_eintr.py b/Lib/test/test_eintr.py index d3cdda0..75452f2 100644 --- a/Lib/test/test_eintr.py +++ b/Lib/test/test_eintr.py @@ -16,7 +16,8 @@ class EINTRTests(unittest.TestCase): # Run the tester in a sub-process, to make sure there is only one # thread (for reliable signal delivery). tester = support.findfile("eintr_tester.py", subdir="eintrdata") - script_helper.assert_python_ok(tester) + # use -u to try to get the full output if the test hangs or crash + script_helper.assert_python_ok("-u", tester) if __name__ == "__main__": -- cgit v0.12 From 29a1445136c7353543b516a085c38b8be9ce5109 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 1 Oct 2015 20:54:41 +0100 Subject: Closes #24884: refactored WatchedFileHandler file reopening into a separate method, based on a suggestion and patch by Marian Horban. --- Doc/library/logging.handlers.rst | 12 +++++++++--- Lib/logging/handlers.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 67403a9..0d3928c 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -162,11 +162,17 @@ for this value. first call to :meth:`emit`. By default, the file grows indefinitely. + .. method:: reopenIfNeeded() + + Checks to see if the file has changed. If it has, the existing stream is + flushed and closed and the file opened again, typically as a precursor to + outputting the record to the file. + + .. method:: emit(record) - Outputs the record to the file, but first checks to see if the file has - changed. If it has, the existing stream is flushed and closed and the - file opened again, before outputting the record to the file. + Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to + reopen the file if it has changed. .. _base-rotating-handler: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 02a5fc1..54bee89 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -440,11 +440,11 @@ class WatchedFileHandler(logging.FileHandler): sres = os.fstat(self.stream.fileno()) self.dev, self.ino = sres[ST_DEV], sres[ST_INO] - def emit(self, record): + def reopenIfNeeded(self): """ - Emit a record. + Reopen log file if needed. - First check if the underlying file has changed, and if it + Checks if the underlying file has changed, and if it has, close the old stream and reopen the file to get the current stream. """ @@ -467,6 +467,15 @@ class WatchedFileHandler(logging.FileHandler): # open a new file handle and get new stat info from that fd self.stream = self._open() self._statstream() + + def emit(self, record): + """ + Emit a record. + + If underlying file has changed, reopen the file before emitting the + record to it. + """ + self.reopenIfNeeded() logging.FileHandler.emit(self, record) -- cgit v0.12 From 01ada3996bded57d1baf9c54b050fc55907d9b13 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 21:54:51 +0200 Subject: Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. Patch co-written with Serhiy Storchaka. --- Doc/whatsnew/3.6.rst | 3 + Lib/test/test_codecs.py | 37 +++++++++--- Misc/NEWS | 4 ++ Objects/stringlib/codecs.h | 147 +++++++++++++++++++++++++++++---------------- Objects/unicodeobject.c | 7 ++- 5 files changed, 135 insertions(+), 63 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 16cdca0..ca83ef9 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -120,6 +120,9 @@ Optimizations * The ASCII and the Latin1 encoders are now up to 3 times as fast for the error error ``surrogateescape``. +* The UTF-8 encoder is now up to 75 times as fast for error handlers: + ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 254c0c1..4658497 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -361,6 +361,12 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual("[\uDC80]".encode(self.encoding, "replace"), "[?]".encode(self.encoding)) + # sequential surrogate characters + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"), + "[]".encode(self.encoding)) + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"), + "[??]".encode(self.encoding)) + bom = "".encode(self.encoding) for before, after in [("\U00010fff", "A"), ("[", "]"), ("A", "\U00010fff")]: @@ -753,6 +759,7 @@ class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" ill_formed_sequence = b"\xed\xb2\x80" ill_formed_sequence_replace = "\ufffd" * 3 + BOM = b'' def test_partial(self): self.check_partial( @@ -785,23 +792,32 @@ class UTF8Test(ReadTest, unittest.TestCase): super().test_lone_surrogates() # not sure if this is making sense for # UTF-16 and UTF-32 - self.assertEqual("[\uDC80]".encode('utf-8', "surrogateescape"), - b'[\x80]') + self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"), + self.BOM + b'[\x80]') + + with self.assertRaises(UnicodeEncodeError) as cm: + "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape") + exc = cm.exception + self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF') def test_surrogatepass_handler(self): - self.assertEqual("abc\ud800def".encode("utf-8", "surrogatepass"), - b"abc\xed\xa0\x80def") - self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), + self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"), + self.BOM + b"abc\xed\xa0\x80def") + self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"), + self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"), + self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]') + + self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"), "abc\ud800def") - self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), - b"\xf0\x90\xbf\xbf\xed\xa0\x80") - self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"), "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0".decode("utf-8", "surrogatepass") + b"abc\xed\xa0".decode(self.encoding, "surrogatepass") with self.assertRaises(UnicodeDecodeError): - b"abc\xed\xa0z".decode("utf-8", "surrogatepass") + b"abc\xed\xa0z".decode(self.encoding, "surrogatepass") @unittest.skipUnless(sys.platform == 'win32', @@ -1008,6 +1024,7 @@ class ReadBufferTest(unittest.TestCase): class UTF8SigTest(UTF8Test, unittest.TestCase): encoding = "utf-8-sig" + BOM = codecs.BOM_UTF8 def test_partial(self): self.check_partial( diff --git a/Misc/NEWS b/Misc/NEWS index 68c0be1..1fae34a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error + handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + Patch co-written with Serhiy Storchaka. + - Issue #25280: Import trace messages emitted in verbose (-v) mode are no longer formatted twice. diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 0fc6b58..562191c 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -268,9 +268,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, Py_ssize_t nallocated; /* number of result bytes allocated */ Py_ssize_t nneeded; /* number of result bytes needed */ #if STRINGLIB_SIZEOF_CHAR > 1 - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; PyObject *rep = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; #endif #if STRINGLIB_SIZEOF_CHAR == 1 const Py_ssize_t max_char_size = 2; @@ -326,72 +327,116 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } #if STRINGLIB_SIZEOF_CHAR > 1 else if (Py_UNICODE_IS_SURROGATE(ch)) { - Py_ssize_t newpos; - Py_ssize_t repsize, k, startpos; + Py_ssize_t startpos, endpos, newpos; + Py_ssize_t repsize, k; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + startpos = i-1; - rep = unicode_encode_call_errorhandler( - errors, &errorHandler, "utf-8", "surrogates not allowed", - unicode, &exc, startpos, startpos+1, &newpos); - if (!rep) - goto error; - - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_LENGTH(rep); + endpos = startpos+1; + + while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos])) + endpos++; + + switch (error_handler) + { + case _Py_ERROR_REPLACE: + memset(p, '?', endpos - startpos); + p += (endpos - startpos); + /* fall through the ignore handler */ + case _Py_ERROR_IGNORE: + i += (endpos - startpos - 1); + break; - if (repsize > max_char_size) { - Py_ssize_t offset; - if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); + case _Py_ERROR_SURROGATEPASS: + for (k=startpos; k> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + i += (endpos - startpos - 1); + break; - if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { - /* integer overflow */ - PyErr_NoMemory(); - goto error; + case _Py_ERROR_SURROGATEESCAPE: + for (k=startpos; k= endpos) { + i += (endpos - startpos - 1); + break; + } + startpos = k; + assert(startpos < endpos); + /* fall through the default handler */ + + default: + rep = unicode_encode_call_errorhandler( + errors, &error_handler_obj, "utf-8", "surrogates not allowed", + unicode, &exc, startpos, endpos, &newpos); + if (!rep) + goto error; + + if (PyBytes_Check(rep)) + repsize = PyBytes_GET_SIZE(rep); + else + repsize = PyUnicode_GET_LENGTH(rep); + + if (repsize > max_char_size) { + Py_ssize_t offset; + if (result == NULL) + offset = p - stackbuf; + else + offset = p - PyBytes_AS_STRING(result); + + if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { + /* integer overflow */ + PyErr_NoMemory(); goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + nallocated += repsize - max_char_size; + if (result != NULL) { + if (_PyBytes_Resize(&result, nallocated) < 0) + goto error; + } else { + result = PyBytes_FromStringAndSize(NULL, nallocated); + if (result == NULL) + goto error; + Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } + p = PyBytes_AS_STRING(result) + offset; } - p = PyBytes_AS_STRING(result) + offset; - } - - if (PyBytes_Check(rep)) { - char *prep = PyBytes_AS_STRING(rep); - for(k = repsize; k > 0; k--) - *p++ = *prep++; - } else /* rep is unicode */ { - enum PyUnicode_Kind repkind; - void *repdata; - if (PyUnicode_READY(rep) < 0) - goto error; - repkind = PyUnicode_KIND(rep); - repdata = PyUnicode_DATA(rep); + if (PyBytes_Check(rep)) { + memcpy(p, PyBytes_AS_STRING(rep), repsize); + p += repsize; + } + else { + /* rep is unicode */ + if (PyUnicode_READY(rep) < 0) + goto error; - for(k=0; k 2 @@ -430,7 +475,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } #if STRINGLIB_SIZEOF_CHAR > 1 - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif return result; @@ -438,7 +483,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); - Py_XDECREF(errorHandler); + Py_XDECREF(error_handler_obj); Py_XDECREF(exc); Py_XDECREF(result); return NULL; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6657cd4..df3a1b5 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -297,6 +297,7 @@ typedef enum { _Py_ERROR_UNKNOWN=0, _Py_ERROR_STRICT, _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_SURROGATEPASS, _Py_ERROR_REPLACE, _Py_ERROR_IGNORE, _Py_ERROR_XMLCHARREFREPLACE, @@ -312,6 +313,8 @@ get_error_handler(const char *errors) return _Py_ERROR_STRICT; if (strcmp(errors, "surrogateescape") == 0) return _Py_ERROR_SURROGATEESCAPE; + if (strcmp(errors, "surrogatepass") == 0) + return _Py_ERROR_SURROGATEPASS; if (strcmp(errors, "ignore") == 0) return _Py_ERROR_IGNORE; if (strcmp(errors, "replace") == 0) @@ -6479,8 +6482,8 @@ unicode_encode_ucs1(PyObject *unicode, goto onError; case _Py_ERROR_REPLACE: - while (collstart++ < collend) - *str++ = '?'; + memset(str, '?', collend - collstart); + str += (collend - collstart); /* fall through ignore error handler */ case _Py_ERROR_IGNORE: pos = collend; -- cgit v0.12 From 13f7fc5771f07d401002dbe7e67ad1a62d0e8ce5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 22:06:54 +0200 Subject: Update importlib_external.h --- Python/importlib_external.h | 113 ++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 56 deletions(-) diff --git a/Python/importlib_external.h b/Python/importlib_external.h index f4f5ac4..c4b6dc9 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -613,7 +613,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, - 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,240, + 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, 9,0,114,31,0,124,2,0,124,4,0,100,2,0,60,110, 6,0,100,3,0,125,2,0,124,3,0,100,1,0,107,9, @@ -621,59 +621,60 @@ const unsigned char _Py_M__importlib_external[] = { 0,100,1,0,100,5,0,133,2,0,25,125,5,0,124,0, 0,100,5,0,100,6,0,133,2,0,25,125,6,0,124,0, 0,100,6,0,100,7,0,133,2,0,25,125,7,0,124,5, - 0,116,0,0,107,3,0,114,168,0,100,8,0,106,1,0, + 0,116,0,0,107,3,0,114,171,0,100,8,0,106,1,0, 124,2,0,124,5,0,131,2,0,125,8,0,116,2,0,106, - 3,0,124,8,0,131,1,0,1,116,4,0,124,8,0,124, - 4,0,141,1,0,130,1,0,110,119,0,116,5,0,124,6, - 0,131,1,0,100,5,0,107,3,0,114,229,0,100,9,0, - 106,1,0,124,2,0,131,1,0,125,8,0,116,2,0,106, - 3,0,124,8,0,131,1,0,1,116,6,0,124,8,0,131, - 1,0,130,1,0,110,58,0,116,5,0,124,7,0,131,1, - 0,100,5,0,107,3,0,114,31,1,100,10,0,106,1,0, - 124,2,0,131,1,0,125,8,0,116,2,0,106,3,0,124, - 8,0,131,1,0,1,116,6,0,124,8,0,131,1,0,130, - 1,0,124,1,0,100,1,0,107,9,0,114,226,1,121,20, - 0,116,7,0,124,1,0,100,11,0,25,131,1,0,125,9, - 0,87,110,18,0,4,116,8,0,107,10,0,114,83,1,1, - 1,1,89,110,62,0,88,116,9,0,124,6,0,131,1,0, - 124,9,0,107,3,0,114,145,1,100,12,0,106,1,0,124, - 2,0,131,1,0,125,8,0,116,2,0,106,3,0,124,8, - 0,131,1,0,1,116,4,0,124,8,0,124,4,0,141,1, - 0,130,1,0,121,18,0,124,1,0,100,13,0,25,100,14, - 0,64,125,10,0,87,110,18,0,4,116,8,0,107,10,0, - 114,183,1,1,1,1,89,110,43,0,88,116,9,0,124,7, - 0,131,1,0,124,10,0,107,3,0,114,226,1,116,4,0, - 100,12,0,106,1,0,124,2,0,131,1,0,124,4,0,141, - 1,0,130,1,0,124,0,0,100,7,0,100,1,0,133,2, - 0,25,83,41,15,97,122,1,0,0,86,97,108,105,100,97, - 116,101,32,116,104,101,32,104,101,97,100,101,114,32,111,102, - 32,116,104,101,32,112,97,115,115,101,100,45,105,110,32,98, - 121,116,101,99,111,100,101,32,97,103,97,105,110,115,116,32, - 115,111,117,114,99,101,95,115,116,97,116,115,32,40,105,102, - 10,32,32,32,32,103,105,118,101,110,41,32,97,110,100,32, - 114,101,116,117,114,110,105,110,103,32,116,104,101,32,98,121, - 116,101,99,111,100,101,32,116,104,97,116,32,99,97,110,32, - 98,101,32,99,111,109,112,105,108,101,100,32,98,121,32,99, - 111,109,112,105,108,101,40,41,46,10,10,32,32,32,32,65, - 108,108,32,111,116,104,101,114,32,97,114,103,117,109,101,110, - 116,115,32,97,114,101,32,117,115,101,100,32,116,111,32,101, - 110,104,97,110,99,101,32,101,114,114,111,114,32,114,101,112, - 111,114,116,105,110,103,46,10,10,32,32,32,32,73,109,112, - 111,114,116,69,114,114,111,114,32,105,115,32,114,97,105,115, - 101,100,32,119,104,101,110,32,116,104,101,32,109,97,103,105, - 99,32,110,117,109,98,101,114,32,105,115,32,105,110,99,111, - 114,114,101,99,116,32,111,114,32,116,104,101,32,98,121,116, - 101,99,111,100,101,32,105,115,10,32,32,32,32,102,111,117, - 110,100,32,116,111,32,98,101,32,115,116,97,108,101,46,32, - 69,79,70,69,114,114,111,114,32,105,115,32,114,97,105,115, - 101,100,32,119,104,101,110,32,116,104,101,32,100,97,116,97, - 32,105,115,32,102,111,117,110,100,32,116,111,32,98,101,10, - 32,32,32,32,116,114,117,110,99,97,116,101,100,46,10,10, - 32,32,32,32,78,114,98,0,0,0,122,10,60,98,121,116, - 101,99,111,100,101,62,114,35,0,0,0,114,12,0,0,0, - 233,8,0,0,0,233,12,0,0,0,122,30,98,97,100,32, - 109,97,103,105,99,32,110,117,109,98,101,114,32,105,110,32, - 123,33,114,125,58,32,123,33,114,125,122,43,114,101,97,99, + 3,0,100,9,0,124,8,0,131,2,0,1,116,4,0,124, + 8,0,124,4,0,141,1,0,130,1,0,110,125,0,116,5, + 0,124,6,0,131,1,0,100,5,0,107,3,0,114,235,0, + 100,10,0,106,1,0,124,2,0,131,1,0,125,8,0,116, + 2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116, + 6,0,124,8,0,131,1,0,130,1,0,110,61,0,116,5, + 0,124,7,0,131,1,0,100,5,0,107,3,0,114,40,1, + 100,11,0,106,1,0,124,2,0,131,1,0,125,8,0,116, + 2,0,106,3,0,100,9,0,124,8,0,131,2,0,1,116, + 6,0,124,8,0,131,1,0,130,1,0,124,1,0,100,1, + 0,107,9,0,114,238,1,121,20,0,116,7,0,124,1,0, + 100,12,0,25,131,1,0,125,9,0,87,110,18,0,4,116, + 8,0,107,10,0,114,92,1,1,1,1,89,110,65,0,88, + 116,9,0,124,6,0,131,1,0,124,9,0,107,3,0,114, + 157,1,100,13,0,106,1,0,124,2,0,131,1,0,125,8, + 0,116,2,0,106,3,0,100,9,0,124,8,0,131,2,0, + 1,116,4,0,124,8,0,124,4,0,141,1,0,130,1,0, + 121,18,0,124,1,0,100,14,0,25,100,15,0,64,125,10, + 0,87,110,18,0,4,116,8,0,107,10,0,114,195,1,1, + 1,1,89,110,43,0,88,116,9,0,124,7,0,131,1,0, + 124,10,0,107,3,0,114,238,1,116,4,0,100,13,0,106, + 1,0,124,2,0,131,1,0,124,4,0,141,1,0,130,1, + 0,124,0,0,100,7,0,100,1,0,133,2,0,25,83,41, + 16,97,122,1,0,0,86,97,108,105,100,97,116,101,32,116, + 104,101,32,104,101,97,100,101,114,32,111,102,32,116,104,101, + 32,112,97,115,115,101,100,45,105,110,32,98,121,116,101,99, + 111,100,101,32,97,103,97,105,110,115,116,32,115,111,117,114, + 99,101,95,115,116,97,116,115,32,40,105,102,10,32,32,32, + 32,103,105,118,101,110,41,32,97,110,100,32,114,101,116,117, + 114,110,105,110,103,32,116,104,101,32,98,121,116,101,99,111, + 100,101,32,116,104,97,116,32,99,97,110,32,98,101,32,99, + 111,109,112,105,108,101,100,32,98,121,32,99,111,109,112,105, + 108,101,40,41,46,10,10,32,32,32,32,65,108,108,32,111, + 116,104,101,114,32,97,114,103,117,109,101,110,116,115,32,97, + 114,101,32,117,115,101,100,32,116,111,32,101,110,104,97,110, + 99,101,32,101,114,114,111,114,32,114,101,112,111,114,116,105, + 110,103,46,10,10,32,32,32,32,73,109,112,111,114,116,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119, + 104,101,110,32,116,104,101,32,109,97,103,105,99,32,110,117, + 109,98,101,114,32,105,115,32,105,110,99,111,114,114,101,99, + 116,32,111,114,32,116,104,101,32,98,121,116,101,99,111,100, + 101,32,105,115,10,32,32,32,32,102,111,117,110,100,32,116, + 111,32,98,101,32,115,116,97,108,101,46,32,69,79,70,69, + 114,114,111,114,32,105,115,32,114,97,105,115,101,100,32,119, + 104,101,110,32,116,104,101,32,100,97,116,97,32,105,115,32, + 102,111,117,110,100,32,116,111,32,98,101,10,32,32,32,32, + 116,114,117,110,99,97,116,101,100,46,10,10,32,32,32,32, + 78,114,98,0,0,0,122,10,60,98,121,116,101,99,111,100, + 101,62,114,35,0,0,0,114,12,0,0,0,233,8,0,0, + 0,233,12,0,0,0,122,30,98,97,100,32,109,97,103,105, + 99,32,110,117,109,98,101,114,32,105,110,32,123,33,114,125, + 58,32,123,33,114,125,122,2,123,125,122,43,114,101,97,99, 104,101,100,32,69,79,70,32,119,104,105,108,101,32,114,101, 97,100,105,110,103,32,116,105,109,101,115,116,97,109,112,32, 105,110,32,123,33,114,125,122,48,114,101,97,99,104,101,100, @@ -699,9 +700,9 @@ const unsigned char _Py_M__importlib_external[] = { 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, - 16,1,16,1,12,1,18,1,13,1,18,1,18,1,15,1, - 13,1,15,1,18,1,15,1,13,1,12,1,12,1,3,1, - 20,1,13,1,5,2,18,1,15,1,13,1,15,1,3,1, + 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, + 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, + 20,1,13,1,5,2,18,1,15,1,16,1,15,1,3,1, 18,1,13,1,5,2,18,1,15,1,9,1,114,135,0,0, 0,99,4,0,0,0,0,0,0,0,5,0,0,0,6,0, 0,0,67,0,0,0,115,115,0,0,0,116,0,0,106,1, -- cgit v0.12 From 3222da26fe873410cbdbc18744e19a7d0f6a7523 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2015 22:07:32 +0200 Subject: Make _PyUnicode_TranslateCharmap() symbol private unicodeobject.h exposes PyUnicode_TranslateCharmap() and PyUnicode_Translate(). --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index df3a1b5..93c4ad9 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8683,7 +8683,7 @@ exit: return res; } -PyObject * +static PyObject * _PyUnicode_TranslateCharmap(PyObject *input, PyObject *mapping, const char *errors) -- cgit v0.12 From 6661d885a34b355200b8ee6bd559af0a5cfc9c8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2015 23:00:39 +0200 Subject: Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check if it is supported, it may not be supported on OpenBSD for example. --- Doc/library/crypt.rst | 2 +- Lib/crypt.py | 3 +-- Misc/NEWS | 4 ++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index b4c90cd..04ffdb2 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -64,7 +64,7 @@ Module Attributes A list of available password hashing algorithms, as ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest, and is guaranteed to have at least ``crypt.METHOD_CRYPT``. + weakest. Module Functions diff --git a/Lib/crypt.py b/Lib/crypt.py index 49ab96e..fbc5f4c 100644 --- a/Lib/crypt.py +++ b/Lib/crypt.py @@ -54,9 +54,8 @@ METHOD_SHA256 = _Method('SHA256', '5', 16, 63) METHOD_SHA512 = _Method('SHA512', '6', 16, 106) methods = [] -for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5): +for _method in (METHOD_SHA512, METHOD_SHA256, METHOD_MD5, METHOD_CRYPT): _result = crypt('', _method) if _result and len(_result) == _method.total_size: methods.append(_method) -methods.append(METHOD_CRYPT) del _result, _method diff --git a/Misc/NEWS b/Misc/NEWS index b43073f..8aad689 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -40,6 +40,10 @@ Core and Builtins Library ------- +- Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not + supported. Check if it is supported, it may not be supported on OpenBSD for + example. + - Issue #23600: Default implementation of tzinfo.fromutc() was returning wrong results in some cases. -- cgit v0.12 From 076fc872bca0f3aa4e6140021e5c05fc4360e82a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:20:56 +0200 Subject: Issue #18174: "python -m test --huntrleaks ..." now also checks for leak of file descriptors. Patch written by Richard Oudkerk. --- Lib/test/libregrtest/refleak.py | 47 ++++++++++++++++++++++++++---- Lib/test/test_regrtest.py | 63 ++++++++++++++++++++++++++++++++--------- Misc/NEWS | 3 ++ 3 files changed, 94 insertions(+), 19 deletions(-) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 9be0dec..59dc49f 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -1,3 +1,4 @@ +import errno import os import re import sys @@ -6,6 +7,36 @@ from inspect import isabstract from test import support +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors""" + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count + + def dash_R(the_module, test, indirect_test, huntrleaks): """Run a test multiple times, looking for reference leaks. @@ -42,20 +73,25 @@ def dash_R(the_module, test, indirect_test, huntrleaks): repcount = nwarmup + ntracked rc_deltas = [0] * repcount alloc_deltas = [0] * repcount + fd_deltas = [0] * repcount print("beginning", repcount, "repetitions", file=sys.stderr) print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, flush=True) # initialize variables to make pyflakes quiet - rc_before = alloc_before = 0 + rc_before = alloc_before = fd_before = 0 for i in range(repcount): indirect_test() - alloc_after, rc_after = dash_R_cleanup(fs, ps, pic, zdc, abcs) + alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, + abcs) print('.', end='', flush=True) if i >= nwarmup: rc_deltas[i] = rc_after - rc_before alloc_deltas[i] = alloc_after - alloc_before - alloc_before, rc_before = alloc_after, rc_after + fd_deltas[i] = fd_after - fd_before + alloc_before = alloc_after + rc_before = rc_after + fd_before = fd_after print(file=sys.stderr) # These checkers return False on success, True on failure def check_rc_deltas(deltas): @@ -71,7 +107,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): failed = False for deltas, item_name, checker in [ (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_alloc_deltas)]: + (alloc_deltas, 'memory blocks', check_alloc_deltas), + (fd_deltas, 'file descriptors', check_rc_deltas)]: if checker(deltas): msg = '%s leaked %s %s, sum=%s' % ( test, deltas[nwarmup:], item_name, sum(deltas)) @@ -151,7 +188,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): func1 = sys.getallocatedblocks func2 = sys.gettotalrefcount gc.collect() - return func1(), func2() + return func1(), func2(), fd_count() def warm_caches(): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f5f22c..8b2449a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -383,27 +383,32 @@ class BaseTestCase(unittest.TestCase): self.assertTrue(0 <= randseed <= 10000000, randseed) return randseed - def run_command(self, args, input=None, exitcode=0): + def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' + if 'stderr' not in kw: + kw['stderr'] = subprocess.PIPE proc = subprocess.run(args, universal_newlines=True, input=input, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + **kw) if proc.returncode != exitcode: - self.fail("Command %s failed with exit code %s\n" - "\n" - "stdout:\n" - "---\n" - "%s\n" - "---\n" - "\n" - "stderr:\n" - "---\n" - "%s" - "---\n" - % (str(args), proc.returncode, proc.stdout, proc.stderr)) + msg = ("Command %s failed with exit code %s\n" + "\n" + "stdout:\n" + "---\n" + "%s\n" + "---\n" + % (str(args), proc.returncode, proc.stdout)) + if proc.stderr: + msg += ("\n" + "stderr:\n" + "---\n" + "%s" + "---\n" + % proc.stderr) + self.fail(msg) return proc @@ -637,6 +642,36 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + def test_huntrleaks_fd_leak(self): + # test --huntrleaks for file descriptor leak + code = textwrap.dedent(""" + import os + import unittest + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + fd = os.open(__file__, os.O_RDONLY) + # bug: never cloes the file descriptor + """) + test = self.create_test(code=code) + + filename = 'reflog.txt' + self.addCleanup(support.unlink, filename) + output = self.run_tests('--huntrleaks', '3:3:', test, + exitcode=1, + stderr=subprocess.STDOUT) + self.check_executed_tests(output, [test], failed=test) + + line = 'beginning 6 repetitions\n123456\n......\n' + self.check_line(output, re.escape(line)) + + line2 = '%s leaked [1, 1, 1] file descriptors, sum=3\n' % test + self.check_line(output, re.escape(line2)) + + with open(filename) as fp: + reflog = fp.read() + self.assertEqual(reflog, line2) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 8aad689..5b708ea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Documentation Tests ----- +- Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of + file descriptors. Patch written by Richard Oudkerk. + - Issue #25260: Fix ``python -m test --coverage`` on Windows. Remove the list of ignored directories. -- cgit v0.12 From 5f9d3acc5e8050a43c79cf4d2a373be67f3c27fc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 00:21:12 +0200 Subject: Issue #22806: Add ``python -m test --list-tests`` command to list tests. --- Lib/test/libregrtest/cmdline.py | 11 +++------ Lib/test/libregrtest/main.py | 55 +++++++++++++++++++++++++++-------------- Lib/test/test_regrtest.py | 7 ++++++ Misc/NEWS | 2 ++ 4 files changed, 49 insertions(+), 26 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index e55e53f..ae1aeb9 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,4 @@ import argparse -import faulthandler import os from test import support @@ -234,6 +233,9 @@ def _create_parser(): group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' 'error happens') + group.add_argument('--list-tests', action='store_true', + help="only write the name of tests that will be run, " + "don't execute them") parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -301,12 +303,7 @@ def _parse_args(args, **kwargs): if ns.quiet: ns.verbose = 0 if ns.timeout is not None: - if hasattr(faulthandler, 'dump_traceback_later'): - if ns.timeout <= 0: - ns.timeout = None - else: - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later") + if ns.timeout <= 0: ns.timeout = None if ns.use_mp is not None: if ns.use_mp <= 0: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c1ce3b1..30d8a59 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import faulthandler import os import platform import random @@ -110,8 +111,13 @@ class Regrtest: def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) + if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): + print("Warning: The timeout option requires " + "faulthandler.dump_traceback_later", file=sys.stderr) + ns.timeout = None + if ns.threshold is not None and gc is None: - print('No GC available, ignore --threshold.') + print('No GC available, ignore --threshold.', file=sys.stderr) ns.threshold = None if ns.findleaks: @@ -122,7 +128,8 @@ class Regrtest: pass #gc.set_debug(gc.DEBUG_SAVEALL) else: - print('No GC available, disabling --findleaks') + print('No GC available, disabling --findleaks', + file=sys.stderr) ns.findleaks = False # Strip .py extensions. @@ -163,20 +170,6 @@ class Regrtest: nottests.add(arg) self.ns.args = [] - # For a partial run, we do not need to clutter the output. - if (self.ns.verbose - or self.ns.header - or not (self.ns.quiet or self.ns.single - or self.tests or self.ns.args)): - # Print basic platform information - print("==", platform.python_implementation(), *sys.version.split()) - print("== ", platform.platform(aliased=True), - "%s-endian" % sys.byteorder) - print("== ", "hash algorithm:", sys.hash_info.algorithm, - "64bit" if sys.maxsize > 2**32 else "32bit") - print("== ", os.getcwd()) - print("Testing with flags:", sys.flags) - # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) if self.ns.testdir: @@ -199,15 +192,18 @@ class Regrtest: del self.selected[:self.selected.index(self.ns.start)] except ValueError: print("Couldn't find starting test (%s), using all tests" - % self.ns.start) + % self.ns.start, file=sys.stderr) if self.ns.randomize: if self.ns.random_seed is None: self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) - print("Using random seed", self.ns.random_seed) random.shuffle(self.selected) + def list_tests(self): + for name in self.selected: + print(name) + def rerun_failed_tests(self): self.ns.verbose = True self.ns.failfast = False @@ -315,6 +311,23 @@ class Regrtest: return def run_tests(self): + # For a partial run, we do not need to clutter the output. + if (self.ns.verbose + or self.ns.header + or not (self.ns.quiet or self.ns.single + or self.tests or self.ns.args)): + # Print basic platform information + print("==", platform.python_implementation(), *sys.version.split()) + print("== ", platform.platform(aliased=True), + "%s-endian" % sys.byteorder) + print("== ", "hash algorithm:", sys.hash_info.algorithm, + "64bit" if sys.maxsize > 2**32 else "32bit") + print("== ", os.getcwd()) + print("Testing with flags:", sys.flags) + + if self.ns.randomize: + print("Using random seed", self.ns.random_seed) + if self.ns.forever: self.tests = self._test_forever(list(self.selected)) self.test_count = '' @@ -359,8 +372,12 @@ class Regrtest: setup_tests(self.ns) self.find_tests(tests) - self.run_tests() + if self.ns.list_tests: + self.list_tests() + sys.exit(0) + + self.run_tests() self.display_result() if self.ns.verbose2 and self.bad: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 8b2449a..e15e724 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -672,6 +672,13 @@ class ArgsTestCase(BaseTestCase): reflog = fp.read() self.assertEqual(reflog, line2) + def test_list_tests(self): + # test --list-tests + tests = [self.create_test() for i in range(5)] + output = self.run_tests('--list-tests', *tests) + self.assertEqual(output.rstrip().splitlines(), + tests) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 5b708ea..6fb0be7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,8 @@ Documentation Tests ----- +- Issue #22806: Add ``python -m test --list-tests`` command to list tests. + - Issue #18174: ``python -m test --huntrleaks ...`` now also checks for leak of file descriptors. Patch written by Richard Oudkerk. -- cgit v0.12 From eb36fdaad8bd38c60973eb69b1307315fa950372 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 01:55:51 +0200 Subject: Fix _PyUnicodeWriter_PrepareKind() Initialize kind to 0 (PyUnicode_WCHAR_KIND) to ensure that _PyUnicodeWriter_PrepareKind() handles correctly read-only buffer: copy the buffer. --- Objects/unicodeobject.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4fd0430..bc98287 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13294,27 +13294,38 @@ unicode_endswith(PyObject *self, Py_LOCAL_INLINE(void) _PyUnicodeWriter_Update(_PyUnicodeWriter *writer) { - if (!writer->readonly) + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + writer->data = PyUnicode_DATA(writer->buffer); + + if (!writer->readonly) { + writer->kind = PyUnicode_KIND(writer->buffer); writer->size = PyUnicode_GET_LENGTH(writer->buffer); + } else { + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); + /* Copy-on-write mode: set buffer size to 0 so * _PyUnicodeWriter_Prepare() will copy (and enlarge) the buffer on * next write. */ writer->size = 0; } - writer->maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); - writer->data = PyUnicode_DATA(writer->buffer); - writer->kind = PyUnicode_KIND(writer->buffer); } void _PyUnicodeWriter_Init(_PyUnicodeWriter *writer) { memset(writer, 0, sizeof(*writer)); -#ifdef Py_DEBUG - writer->kind = 5; /* invalid kind */ -#endif + + /* ASCII is the bare minimum */ writer->min_char = 127; + + /* use a value smaller than PyUnicode_1BYTE_KIND() so + _PyUnicodeWriter_PrepareKind() will copy the buffer. */ + writer->kind = PyUnicode_WCHAR_KIND; + assert(writer->kind <= PyUnicode_1BYTE_KIND); } int -- cgit v0.12 From 46d75353c5688e0273ca90efcbdb61687c085a80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 02:21:35 +0200 Subject: Issue #18174: Fix test_regrtest when Python is compiled in release mode --- Lib/test/test_regrtest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e15e724..e0307a8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -642,6 +642,7 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) + @unittest.skipUnless(Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v0.12 From fd265f4a1892f7be64ceccadb276e37b3fb61f61 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 2 Oct 2015 23:17:33 -0700 Subject: Factor out common iterator finalization code --- Modules/_collectionsmodule.c | 45 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0c64713..9312636 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -350,21 +350,34 @@ deque_appendleft(dequeobject *deque, PyObject *item) PyDoc_STRVAR(appendleft_doc, "Add an element to the left side of the deque."); +static PyObject* +finalize_iterator(PyObject *it) +{ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) + PyErr_Clear(); + else { + Py_DECREF(it); + return NULL; + } + } + Py_DECREF(it); + Py_RETURN_NONE; +} /* Run an iterator to exhaustion. Shortcut for the extend/extendleft methods when maxlen == 0. */ static PyObject* consume_iterator(PyObject *it) { + PyObject *(*iternext)(PyObject *); PyObject *item; - while ((item = PyIter_Next(it)) != NULL) { + iternext = *Py_TYPE(it)->tp_iternext; + while ((item = iternext(it)) != NULL) { Py_DECREF(item); } - Py_DECREF(it); - if (PyErr_Occurred()) - return NULL; - Py_RETURN_NONE; + return finalize_iterator(it); } static PyObject * @@ -423,16 +436,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (trim) deque_trim_left(deque); } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else { - Py_DECREF(it); - return NULL; - } - } - Py_DECREF(it); - Py_RETURN_NONE; + return finalize_iterator(it); } PyDoc_STRVAR(extend_doc, @@ -494,16 +498,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (trim) deque_trim_right(deque); } - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else { - Py_DECREF(it); - return NULL; - } - } - Py_DECREF(it); - Py_RETURN_NONE; + return finalize_iterator(it); } PyDoc_STRVAR(extendleft_doc, -- cgit v0.12 From 909a950ffc279d463d916ac03c10fb4168ccc007 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 3 Oct 2015 06:25:43 +0000 Subject: Issues #25232, #24657: Add NEWS to 3.6.0a1 section --- Misc/NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 754041f..42d7a86 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -40,6 +40,12 @@ Core and Builtins Library ------- +- Issue #25232: Fix CGIRequestHandler to split the query from the URL at the + first question mark (?) rather than the last. Patch from Xiang Zhang. + +- Issue #24657: Prevent CGIRequestHandler from collapsing slashes in the + query part of the URL as if it were a path. Patch from Xiang Zhang. + - Issue #25287: Don't add crypt.METHOD_CRYPT to crypt.methods if it's not supported. Check if it is supported, it may not be supported on OpenBSD for example. -- cgit v0.12 From 33c30131db0249dec166fcbe22cd9152b405e170 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:20:41 +0200 Subject: Issue #25306: Skip test_huntrleaks_fd_leak() of test_regrtest until the bug is fixed. --- Lib/test/test_regrtest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e0307a8..edbb4b0 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,6 +643,8 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') + # Issue #25306: the test hangs sometimes on Windows + @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" -- cgit v0.12 From 20d2118248f9313a9b77bb4d299252da7fc7fa2f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 3 Oct 2015 21:40:21 +0200 Subject: Issue #25306: Try to fix test_huntrleaks_fd_leak() on Windows Issue #25306: Disable popup and logs to stderr on assertion failures in MSCRT. --- Lib/test/test_regrtest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index edbb4b0..2e33555 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -643,14 +643,24 @@ class ArgsTestCase(BaseTestCase): self.check_executed_tests(output, [test]*3, failed=test) @unittest.skipUnless(Py_DEBUG, 'need a debug build') - # Issue #25306: the test hangs sometimes on Windows - @unittest.skipIf(sys.platform == 'win32', 'test broken on Windows') def test_huntrleaks_fd_leak(self): # test --huntrleaks for file descriptor leak code = textwrap.dedent(""" import os import unittest + # Issue #25306: Disable popups and logs to stderr on assertion + # failures in MSCRT + try: + import msvcrt + msvcrt.CrtSetReportMode + except (ImportError, AttributeError): + # no Windows, o release build + pass + else: + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + msvcrt.CrtSetReportMode(m, 0) + class FDLeakTest(unittest.TestCase): def test_leak(self): fd = os.open(__file__, os.O_RDONLY) -- cgit v0.12 From 1d65d9192dac57776693c55a9ccefbde2ca74c23 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 5 Oct 2015 13:43:50 +0200 Subject: Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error handlers: ``ignore``, ``replace`` and ``surrogateescape``. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/test/test_codecs.py | 12 ++++++++++++ Misc/NEWS | 3 +++ Objects/unicodeobject.c | 48 +++++++++++++++++++++++++++++++++++++++--------- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index ca83ef9..24fd822 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -123,6 +123,9 @@ Optimizations * The UTF-8 encoder is now up to 75 times as fast for error handlers: ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. +* The UTF-8 decoder is now up to 15 times as fast for error handlers: + ``ignore``, ``replace`` and ``surrogateescape``. + Build and C API Changes ======================= diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index bdc331e..7b6883f 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -788,6 +788,18 @@ class UTF8Test(ReadTest, unittest.TestCase): self.check_state_handling_decode(self.encoding, u, u.encode(self.encoding)) + def test_decode_error(self): + for data, error_handler, expected in ( + (b'[\x80\xff]', 'ignore', '[]'), + (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'), + (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'), + (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'), + ): + with self.subTest(data=data, error_handler=error_handler, + expected=expected): + self.assertEqual(data.decode(self.encoding, error_handler), + expected) + def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for diff --git a/Misc/NEWS b/Misc/NEWS index d809377..3991d6b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +* Issue #25301: The UTF-8 decoder is now up to 15 times as fast for error + handlers: ``ignore``, ``replace`` and ``surrogateescape``. + - Issue #24848: Fixed a number of bugs in UTF-7 decoding of misformed data. - Issue #25267: The UTF-8 encoder is now up to 75 times as fast for error diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index bc98287..56614e6 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4714,8 +4714,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t startinpos; Py_ssize_t endinpos; const char *errmsg = ""; - PyObject *errorHandler = NULL; + PyObject *error_handler_obj = NULL; PyObject *exc = NULL; + _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; if (size == 0) { if (consumed) @@ -4740,6 +4741,7 @@ PyUnicode_DecodeUTF8Stateful(const char *s, while (s < end) { Py_UCS4 ch; int kind = writer.kind; + if (kind == PyUnicode_1BYTE_KIND) { if (PyUnicode_IS_ASCII(writer.buffer)) ch = asciilib_utf8_decode(&s, end, writer.data, &writer.pos); @@ -4778,24 +4780,52 @@ PyUnicode_DecodeUTF8Stateful(const char *s, continue; } - if (unicode_decode_call_errorhandler_writer( - errors, &errorHandler, - "utf-8", errmsg, - &starts, &end, &startinpos, &endinpos, &exc, &s, - &writer)) - goto onError; + if (error_handler == _Py_ERROR_UNKNOWN) + error_handler = get_error_handler(errors); + + switch (error_handler) { + case _Py_ERROR_IGNORE: + s += (endinpos - startinpos); + break; + + case _Py_ERROR_REPLACE: + if (_PyUnicodeWriter_WriteCharInline(&writer, 0xfffd) < 0) + goto onError; + s += (endinpos - startinpos); + break; + + case _Py_ERROR_SURROGATEESCAPE: + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) + goto onError; + for (Py_ssize_t i=startinpos; i Date: Mon, 5 Oct 2015 13:49:26 +0200 Subject: Issue #25301: Fix compatibility with ISO C90 --- Objects/unicodeobject.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 56614e6..3d78404 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4795,9 +4795,12 @@ PyUnicode_DecodeUTF8Stateful(const char *s, break; case _Py_ERROR_SURROGATEESCAPE: + { + Py_ssize_t i; + if (_PyUnicodeWriter_PrepareKind(&writer, PyUnicode_2BYTE_KIND) < 0) goto onError; - for (Py_ssize_t i=startinpos; i Date: Mon, 5 Oct 2015 22:52:37 -0400 Subject: Eliminate unnecessary test --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 9312636..4ea9140 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -530,7 +530,7 @@ deque_copy(PyObject *deque) return NULL; new_deque->maxlen = old_deque->maxlen; /* Fast path for the deque_repeat() common case where len(deque) == 1 */ - if (Py_SIZE(deque) == 1 && new_deque->maxlen != 0) { + if (Py_SIZE(deque) == 1) { PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; rv = deque_append(new_deque, item); } else { -- cgit v0.12 From 68713e41a5df7228e6971e894d8fc2fda720b5ce Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Tue, 6 Oct 2015 13:29:56 -0400 Subject: Closes issue #12006: Add ISO 8601 year, week, and day directives to strptime. This commit adds %G, %V, and %u directives to strptime. Thanks Ashley Anderson for the implementation. --- Doc/library/datetime.rst | 37 +++++++++++++++++++++- Doc/whatsnew/3.6.rst | 8 +++++ Lib/_strptime.py | 81 +++++++++++++++++++++++++++++++++++++++-------- Lib/test/test_strptime.py | 57 +++++++++++++++++++++++++-------- Misc/NEWS | 3 ++ 5 files changed, 159 insertions(+), 27 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 5c659e7..cf5d5b8 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1909,6 +1909,34 @@ format codes. | ``%%`` | A literal ``'%'`` character. | % | | +-----------+--------------------------------+------------------------+-------+ +Several additional directives not required by the C89 standard are included for +convenience. These parameters all correspond to ISO 8601 date values. These +may not be available on all platforms when used with the :meth:`strftime` +method. The ISO 8601 year and ISO 8601 week directives are not interchangeable +with the year and week number directives above. Calling :meth:`strptime` with +incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. + ++-----------+--------------------------------+------------------------+-------+ +| Directive | Meaning | Example | Notes | ++===========+================================+========================+=======+ +| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | +| | representing the year that | 2014, ..., 9998, 9999 | | +| | contains the greater part of | | | +| | the ISO week (``%V``). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | +| | number where 1 is Monday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8) | +| | number with Monday as | | | +| | the first day of the week. | | | +| | Week 01 is the week containing | | | +| | Jan 4. | | | ++-----------+--------------------------------+------------------------+-------+ + +.. versionadded:: 3.6 + ``%G``, ``%u`` and ``%V`` were added. + Notes: (1) @@ -1973,7 +2001,14 @@ Notes: (7) When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used - in calculations when the day of the week and the year are specified. + in calculations when the day of the week and the calendar year (``%Y``) + are specified. + +(8) + Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the + day of the week and the ISO year (``%G``) are specified in a + :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not + interchangable. .. rubric:: Footnotes diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 24fd822..dd35c9a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -110,6 +110,14 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) +datetime +-------- + +* :meth:`datetime.stftime ` and + :meth:`date.stftime ` methods now support ISO 8601 + date directives ``%G``, ``%u`` and ``%V``. + (Contributed by Ashley Anderson in :issue:`12006`.) + Optimizations ============= diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 374923d..fe5b046 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -195,12 +195,15 @@ class TimeRE(dict): 'f': r"(?P[0-9]{1,6})", 'H': r"(?P2[0-3]|[0-1]\d|\d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", 'M': r"(?P[0-5]\d|\d)", 'S': r"(?P6[0-1]|[0-5]\d|\d)", 'U': r"(?P5[0-3]|[0-4]\d|\d)", 'w': r"(?P[0-6])", + 'u': r"(?P[1-7])", + 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", #XXX: Does 'Y' need to worry about having less or more than @@ -295,6 +298,22 @@ def _calc_julian_from_U_or_W(year, week_of_year, day_of_week, week_starts_Mon): return 1 + days_to_week + day_of_week +def _calc_julian_from_V(iso_year, iso_week, iso_weekday): + """Calculate the Julian day based on the ISO 8601 year, week, and weekday. + ISO weeks start on Mondays, with week 01 being the week containing 4 Jan. + ISO week days range from 1 (Monday) to 7 (Sunday). + """ + correction = datetime_date(iso_year, 1, 4).isoweekday() + 3 + ordinal = (iso_week * 7) + iso_weekday - correction + # ordinal may be negative or 0 now, which means the date is in the previous + # calendar year + if ordinal < 1: + ordinal += datetime_date(iso_year, 1, 1).toordinal() + iso_year -= 1 + ordinal -= datetime_date(iso_year, 1, 1).toordinal() + return iso_year, ordinal + + def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): """Return a 2-tuple consisting of a time struct and an int containing the number of microseconds based on the input string and the @@ -339,15 +358,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = None + iso_year = year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 tzoffset = None # Default to -1 to signify that values not known; not critical to have, # though - week_of_year = -1 - week_of_year_start = -1 + iso_week = week_of_year = None + week_of_year_start = None # weekday and julian defaulted to None so as to signal need to calculate # values weekday = julian = None @@ -369,6 +388,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) + elif group_key == 'G': + iso_year = int(found_dict['G']) elif group_key == 'm': month = int(found_dict['m']) elif group_key == 'B': @@ -414,6 +435,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): weekday = 6 else: weekday -= 1 + elif group_key == 'u': + weekday = int(found_dict['u']) + weekday -= 1 elif group_key == 'j': julian = int(found_dict['j']) elif group_key in ('U', 'W'): @@ -424,6 +448,8 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: # W starts week on Monday. week_of_year_start = 0 + elif group_key == 'V': + iso_week = int(found_dict['V']) elif group_key == 'z': z = found_dict['z'] tzoffset = int(z[1:3]) * 60 + int(z[3:5]) @@ -444,28 +470,57 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): else: tz = value break + # Deal with the cases where ambiguities arize + # don't assume default values for ISO week/year + if year is None and iso_year is not None: + if iso_week is None or weekday is None: + raise ValueError("ISO year directive '%G' must be used with " + "the ISO week directive '%V' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + if julian is not None: + raise ValueError("Day of the year directive '%j' is not " + "compatible with ISO year directive '%G'. " + "Use '%Y' instead.") + elif week_of_year is None and iso_week is not None: + if weekday is None: + raise ValueError("ISO week directive '%V' must be used with " + "the ISO year directive '%G' and a weekday " + "directive ('%A', '%a', '%w', or '%u').") + else: + raise ValueError("ISO week directive '%V' is incompatible with " + "the year directive '%Y'. Use the ISO year '%G' " + "instead.") + leap_year_fix = False if year is None and month == 2 and day == 29: year = 1904 # 1904 is first leap year of 20th century leap_year_fix = True elif year is None: year = 1900 + + # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian is None and week_of_year != -1 and weekday is not None: - week_starts_Mon = True if week_of_year_start == 0 else False - julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, - week_starts_Mon) - # Cannot pre-calculate datetime_date() since can change in Julian - # calculation and thus could have different value for the day of the week - # calculation. + if julian is None and weekday is not None: + if week_of_year is not None: + week_starts_Mon = True if week_of_year_start == 0 else False + julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, + week_starts_Mon) + elif iso_year is not None and iso_week is not None: + year, julian = _calc_julian_from_V(iso_year, iso_week, weekday + 1) + if julian is None: + # Cannot pre-calculate datetime_date() since can change in Julian + # calculation and thus could have different value for the day of + # the week calculation. # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 - else: # Assume that if they bothered to include Julian day it will - # be accurate. - datetime_result = datetime_date.fromordinal((julian - 1) + datetime_date(year, 1, 1).toordinal()) + else: # Assume that if they bothered to include Julian day (or if it was + # calculated above with year/week/weekday) it will be accurate. + datetime_result = datetime_date.fromordinal( + (julian - 1) + + datetime_date(year, 1, 1).toordinal()) year = datetime_result.year month = datetime_result.month day = datetime_result.day diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 346e2c6..6b26c8a 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -152,8 +152,8 @@ class TimeRETests(unittest.TestCase): "'%s' using '%s'; group 'a' = '%s', group 'b' = %s'" % (found.string, found.re.pattern, found.group('a'), found.group('b'))) - for directive in ('a','A','b','B','c','d','H','I','j','m','M','p','S', - 'U','w','W','x','X','y','Y','Z','%'): + for directive in ('a','A','b','B','c','d','G','H','I','j','m','M','p', + 'S','u','U','V','w','W','x','X','y','Y','Z','%'): compiled = self.time_re.compile("%" + directive) found = compiled.match(time.strftime("%" + directive)) self.assertTrue(found, "Matching failed on '%s' using '%s' regex" % @@ -218,6 +218,26 @@ class StrptimeTests(unittest.TestCase): else: self.fail("'%s' did not raise ValueError" % bad_format) + # Ambiguous or incomplete cases using ISO year/week/weekday directives + # 1. ISO week (%V) is specified, but the year is specified with %Y + # instead of %G + with self.assertRaises(ValueError): + _strptime._strptime("1999 50", "%Y %V") + # 2. ISO year (%G) and ISO week (%V) are specified, but weekday is not + with self.assertRaises(ValueError): + _strptime._strptime("1999 51", "%G %V") + # 3. ISO year (%G) and weekday are specified, but ISO week (%V) is not + for w in ('A', 'a', 'w', 'u'): + with self.assertRaises(ValueError): + _strptime._strptime("1999 51","%G %{}".format(w)) + # 4. ISO year is specified alone (e.g. time.strptime('2015', '%G')) + with self.assertRaises(ValueError): + _strptime._strptime("2015", "%G") + # 5. Julian/ordinal day (%j) is specified with %G, but not %Y + with self.assertRaises(ValueError): + _strptime._strptime("1999 256", "%G %j") + + def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: @@ -289,7 +309,7 @@ class StrptimeTests(unittest.TestCase): def test_weekday(self): # Test weekday directives - for directive in ('A', 'a', 'w'): + for directive in ('A', 'a', 'w', 'u'): self.helper(directive,6) def test_julian(self): @@ -458,16 +478,20 @@ class CalculationTests(unittest.TestCase): # Should be able to infer date if given year, week of year (%U or %W) # and day of the week def test_helper(ymd_tuple, test_reason): - for directive in ('W', 'U'): - format_string = "%%Y %%%s %%w" % directive - dt_date = datetime_date(*ymd_tuple) - strp_input = dt_date.strftime(format_string) - strp_output = _strptime._strptime_time(strp_input, format_string) - self.assertTrue(strp_output[:3] == ymd_tuple, - "%s(%s) test failed w/ '%s': %s != %s (%s != %s)" % - (test_reason, directive, strp_input, - strp_output[:3], ymd_tuple, - strp_output[7], dt_date.timetuple()[7])) + for year_week_format in ('%Y %W', '%Y %U', '%G %V'): + for weekday_format in ('%w', '%u', '%a', '%A'): + format_string = year_week_format + ' ' + weekday_format + with self.subTest(test_reason, + date=ymd_tuple, + format=format_string): + dt_date = datetime_date(*ymd_tuple) + strp_input = dt_date.strftime(format_string) + strp_output = _strptime._strptime_time(strp_input, + format_string) + msg = "%r: %s != %s" % (strp_input, + strp_output[7], + dt_date.timetuple()[7]) + self.assertEqual(strp_output[:3], ymd_tuple, msg) test_helper((1901, 1, 3), "week 0") test_helper((1901, 1, 8), "common case") test_helper((1901, 1, 13), "day on Sunday") @@ -499,18 +523,25 @@ class CalculationTests(unittest.TestCase): self.assertEqual(_strptime._strptime_time(value, format)[:-1], expected) check('2015 0 0', '%Y %U %w', 2014, 12, 28, 0, 0, 0, 6, -3) check('2015 0 0', '%Y %W %w', 2015, 1, 4, 0, 0, 0, 6, 4) + check('2015 1 1', '%G %V %u', 2014, 12, 29, 0, 0, 0, 0, 363) check('2015 0 1', '%Y %U %w', 2014, 12, 29, 0, 0, 0, 0, -2) check('2015 0 1', '%Y %W %w', 2014, 12, 29, 0, 0, 0, 0, -2) + check('2015 1 2', '%G %V %u', 2014, 12, 30, 0, 0, 0, 1, 364) check('2015 0 2', '%Y %U %w', 2014, 12, 30, 0, 0, 0, 1, -1) check('2015 0 2', '%Y %W %w', 2014, 12, 30, 0, 0, 0, 1, -1) + check('2015 1 3', '%G %V %u', 2014, 12, 31, 0, 0, 0, 2, 365) check('2015 0 3', '%Y %U %w', 2014, 12, 31, 0, 0, 0, 2, 0) check('2015 0 3', '%Y %W %w', 2014, 12, 31, 0, 0, 0, 2, 0) + check('2015 1 4', '%G %V %u', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %U %w', 2015, 1, 1, 0, 0, 0, 3, 1) check('2015 0 4', '%Y %W %w', 2015, 1, 1, 0, 0, 0, 3, 1) + check('2015 1 5', '%G %V %u', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %U %w', 2015, 1, 2, 0, 0, 0, 4, 2) check('2015 0 5', '%Y %W %w', 2015, 1, 2, 0, 0, 0, 4, 2) + check('2015 1 6', '%G %V %u', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %U %w', 2015, 1, 3, 0, 0, 0, 5, 3) check('2015 0 6', '%Y %W %w', 2015, 1, 3, 0, 0, 0, 5, 3) + check('2015 1 7', '%G %V %u', 2015, 1, 4, 0, 0, 0, 6, 4) class CacheTests(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index dec38ba..074577f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -383,6 +383,9 @@ Library - Issue #23572: Fixed functools.singledispatch on classes with falsy metaclasses. Patch by Ethan Furman. +- Issue #12006: Add ISO 8601 year, week, and day directives (%G, %V, %u) to + strptime. + Documentation ------------- -- cgit v0.12 From 3c5f0030238125a5a2e87d713e6d6c4a22b945ae Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 6 Oct 2015 21:17:02 -0700 Subject: make configure executable --- configure | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 configure diff --git a/configure b/configure old mode 100644 new mode 100755 -- cgit v0.12 From 6f038ada5bd1bc2838e20f0b22de1780e7ce709d Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 7 Oct 2015 07:54:23 +0300 Subject: Add a versionadded directive for reopenIfNeeded() --- Doc/library/logging.handlers.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 0d3928c..c830efd 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -168,6 +168,8 @@ for this value. flushed and closed and the file opened again, typically as a precursor to outputting the record to the file. + .. versionadded:: 3.6 + .. method:: emit(record) -- cgit v0.12 From c2432f6edbec7a8cc1021d2f4fe625ba92dc8ac4 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 7 Oct 2015 11:15:15 +0000 Subject: One more typo in a comment --- Python/ast.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ast.c b/Python/ast.c index 88fcd37..5a7a745 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4035,7 +4035,7 @@ fstring_compile_expr(PyObject *str, Py_ssize_t expr_start, assert(expr_end >= 0 && expr_end < PyUnicode_GET_LENGTH(str)); assert(expr_end >= expr_start); - /* There has to be at least on character on each side of the + /* There has to be at least one character on each side of the expression inside this str. This will have been caught before we're called. */ assert(expr_start >= 1); -- cgit v0.12 From 960e848f0d32399824d61dce2ff5736bb596ed44 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 8 Oct 2015 12:27:06 +0300 Subject: Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate extensions. Patch by Nikolay Bogoychev. --- Doc/library/urllib.robotparser.rst | 30 ++++++++++++- Doc/whatsnew/3.6.rst | 8 ++++ Lib/test/test_robotparser.py | 90 +++++++++++++++++++++++++++++--------- Lib/urllib/robotparser.py | 39 ++++++++++++++++- Misc/ACKS | 1 + Misc/NEWS | 5 ++- 6 files changed, 147 insertions(+), 26 deletions(-) diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index f179de2..c2e1bef 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -53,15 +53,41 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. Sets the time the ``robots.txt`` file was last fetched to the current time. + .. method:: crawl_delay(useragent) -The following example demonstrates basic use of the RobotFileParser class. + Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` + for the *useragent* in question. If there is no such parameter or it + doesn't apply to the *useragent* specified or the ``robots.txt`` entry + for this parameter has invalid syntax, return ``None``. + + .. versionadded:: 3.6 + + .. method:: request_rate(useragent) + + Returns the contents of the ``Request-rate`` parameter from + ``robots.txt`` in the form of a :func:`~collections.namedtuple` + ``(requests, seconds)``. If there is no such parameter or it doesn't + apply to the *useragent* specified or the ``robots.txt`` entry for this + parameter has invalid syntax, return ``None``. + + .. versionadded:: 3.6 + + +The following example demonstrates basic use of the :class:`RobotFileParser` +class:: >>> import urllib.robotparser >>> rp = urllib.robotparser.RobotFileParser() >>> rp.set_url("http://www.musi-cal.com/robots.txt") >>> rp.read() + >>> rrate = rp.request_rate("*") + >>> rrate.requests + 3 + >>> rrate.seconds + 20 + >>> rp.crawl_delay("*") + 6 >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") False >>> rp.can_fetch("*", "http://www.musi-cal.com/") True - diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index dd35c9a..3080820 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -119,6 +119,14 @@ datetime (Contributed by Ashley Anderson in :issue:`12006`.) +urllib.robotparser +------------------ + +:class:`~urllib.robotparser.RobotFileParser` now supports ``Crawl-delay`` and +``Request-rate`` extensions. +(Contributed by Nikolay Bogoychev in :issue:`16099`.) + + Optimizations ============= diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index d01266f..90b3072 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -1,6 +1,7 @@ import io import unittest import urllib.robotparser +from collections import namedtuple from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support @@ -12,7 +13,8 @@ except ImportError: class RobotTestCase(unittest.TestCase): - def __init__(self, index=None, parser=None, url=None, good=None, agent=None): + def __init__(self, index=None, parser=None, url=None, good=None, + agent=None, request_rate=None, crawl_delay=None): # workaround to make unittest discovery work (see #17066) if not isinstance(index, int): return @@ -25,6 +27,8 @@ class RobotTestCase(unittest.TestCase): self.url = url self.good = good self.agent = agent + self.request_rate = request_rate + self.crawl_delay = crawl_delay def runTest(self): if isinstance(self.url, tuple): @@ -34,6 +38,18 @@ class RobotTestCase(unittest.TestCase): agent = self.agent if self.good: self.assertTrue(self.parser.can_fetch(agent, url)) + self.assertEqual(self.parser.crawl_delay(agent), self.crawl_delay) + # if we have actual values for request rate + if self.request_rate and self.parser.request_rate(agent): + self.assertEqual( + self.parser.request_rate(agent).requests, + self.request_rate.requests + ) + self.assertEqual( + self.parser.request_rate(agent).seconds, + self.request_rate.seconds + ) + self.assertEqual(self.parser.request_rate(agent), self.request_rate) else: self.assertFalse(self.parser.can_fetch(agent, url)) @@ -43,15 +59,17 @@ class RobotTestCase(unittest.TestCase): tests = unittest.TestSuite() def RobotTest(index, robots_txt, good_urls, bad_urls, - agent="test_robotparser"): + request_rate, crawl_delay, agent="test_robotparser"): lines = io.StringIO(robots_txt).readlines() parser = urllib.robotparser.RobotFileParser() parser.parse(lines) for url in good_urls: - tests.addTest(RobotTestCase(index, parser, url, 1, agent)) + tests.addTest(RobotTestCase(index, parser, url, 1, agent, + request_rate, crawl_delay)) for url in bad_urls: - tests.addTest(RobotTestCase(index, parser, url, 0, agent)) + tests.addTest(RobotTestCase(index, parser, url, 0, agent, + request_rate, crawl_delay)) # Examples from http://www.robotstxt.org/wc/norobots.html (fetched 2002) @@ -65,14 +83,18 @@ Disallow: /foo.html good = ['/','/test.html'] bad = ['/cyberworld/map/index.html','/tmp/xxx','/foo.html'] +request_rate = None +crawl_delay = None -RobotTest(1, doc, good, bad) +RobotTest(1, doc, good, bad, request_rate, crawl_delay) # 2. doc = """ # robots.txt for http://www.example.com/ User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. @@ -83,8 +105,10 @@ Disallow: good = ['/','/test.html',('cybermapper','/cyberworld/map/index.html')] bad = ['/cyberworld/map/index.html'] +request_rate = None # The parameters should be equal to None since they +crawl_delay = None # don't apply to the cybermapper user agent -RobotTest(2, doc, good, bad) +RobotTest(2, doc, good, bad, request_rate, crawl_delay) # 3. doc = """ @@ -95,14 +119,18 @@ Disallow: / good = [] bad = ['/cyberworld/map/index.html','/','/tmp/'] +request_rate = None +crawl_delay = None -RobotTest(3, doc, good, bad) +RobotTest(3, doc, good, bad, request_rate, crawl_delay) # Examples from http://www.robotstxt.org/wc/norobots-rfc.html (fetched 2002) # 4. doc = """ User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html @@ -115,8 +143,17 @@ bad = ['/tmp','/tmp.html','/tmp/a.html', '/~joe/index.html' ] -RobotTest(4, doc, good, bad, 'figtree') -RobotTest(5, doc, good, bad, 'FigTree Robot libwww-perl/5.04') +request_rate = namedtuple('req_rate', 'requests seconds') +request_rate.requests = 9 +request_rate.seconds = 30 +crawl_delay = 3 +request_rate_bad = None # not actually tested, but we still need to parse it +crawl_delay_bad = None # in order to accommodate the input parameters + + +RobotTest(4, doc, good, bad, request_rate, crawl_delay, 'figtree' ) +RobotTest(5, doc, good, bad, request_rate_bad, crawl_delay_bad, + 'FigTree Robot libwww-perl/5.04') # 6. doc = """ @@ -125,14 +162,18 @@ Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html +Crawl-delay: 3 +Request-rate: 9/banana """ good = ['/tmp',] # XFAIL: '/a%2fb.html' bad = ['/tmp/','/tmp/a.html', '/a%3cd.html','/a%3Cd.html',"/a/b.html", '/%7Ejoe/index.html'] +crawl_delay = 3 +request_rate = None # since request rate has invalid syntax, return None -RobotTest(6, doc, good, bad) +RobotTest(6, doc, good, bad, None, None) # From bug report #523041 @@ -140,12 +181,16 @@ RobotTest(6, doc, good, bad) doc = """ User-Agent: * Disallow: /. +Crawl-delay: pears """ good = ['/foo.html'] -bad = [] # Bug report says "/" should be denied, but that is not in the RFC +bad = [] # bug report says "/" should be denied, but that is not in the RFC + +crawl_delay = None # since crawl delay has invalid syntax, return None +request_rate = None -RobotTest(7, doc, good, bad) +RobotTest(7, doc, good, bad, crawl_delay, request_rate) # From Google: http://www.google.com/support/webmasters/bin/answer.py?hl=en&answer=40364 @@ -154,12 +199,15 @@ doc = """ User-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ +Request-rate: whale/banana """ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] +crawl_delay = None +request_rate = None # invalid syntax, return none -RobotTest(8, doc, good, bad, agent="Googlebot") +RobotTest(8, doc, good, bad, crawl_delay, request_rate, agent="Googlebot") # 9. This file is incorrect because "Googlebot" is a substring of # "Googlebot-Mobile", so test 10 works just like test 9. @@ -174,12 +222,12 @@ Allow: / good = [] bad = ['/something.jpg'] -RobotTest(9, doc, good, bad, agent="Googlebot") +RobotTest(9, doc, good, bad, None, None, agent="Googlebot") good = [] bad = ['/something.jpg'] -RobotTest(10, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(10, doc, good, bad, None, None, agent="Googlebot-Mobile") # 11. Get the order correct. doc = """ @@ -193,12 +241,12 @@ Disallow: / good = [] bad = ['/something.jpg'] -RobotTest(11, doc, good, bad, agent="Googlebot") +RobotTest(11, doc, good, bad, None, None, agent="Googlebot") good = ['/something.jpg'] bad = [] -RobotTest(12, doc, good, bad, agent="Googlebot-Mobile") +RobotTest(12, doc, good, bad, None, None, agent="Googlebot-Mobile") # 13. Google also got the order wrong in #8. You need to specify the @@ -212,7 +260,7 @@ Disallow: /folder1/ good = ['/folder1/myfile.html'] bad = ['/folder1/anotherfile.html'] -RobotTest(13, doc, good, bad, agent="googlebot") +RobotTest(13, doc, good, bad, None, None, agent="googlebot") # 14. For issue #6325 (query string support) @@ -224,7 +272,7 @@ Disallow: /some/path?name=value good = ['/some/path'] bad = ['/some/path?name=value'] -RobotTest(14, doc, good, bad) +RobotTest(14, doc, good, bad, None, None) # 15. For issue #4108 (obey first * entry) doc = """ @@ -238,7 +286,7 @@ Disallow: /another/path good = ['/another/path'] bad = ['/some/path'] -RobotTest(15, doc, good, bad) +RobotTest(15, doc, good, bad, None, None) # 16. Empty query (issue #17403). Normalizing the url first. doc = """ @@ -250,7 +298,7 @@ Disallow: /another/path? good = ['/some/path?'] bad = ['/another/path?'] -RobotTest(16, doc, good, bad) +RobotTest(16, doc, good, bad, None, None) class RobotHandler(BaseHTTPRequestHandler): diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 4fbb0cb..4ac553a 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -10,7 +10,9 @@ http://www.robotstxt.org/norobots-rfc.txt """ -import urllib.parse, urllib.request +import collections +import urllib.parse +import urllib.request __all__ = ["RobotFileParser"] @@ -120,10 +122,29 @@ class RobotFileParser: if state != 0: entry.rulelines.append(RuleLine(line[1], True)) state = 2 + elif line[0] == "crawl-delay": + if state != 0: + # before trying to convert to int we need to make + # sure that robots.txt has valid syntax otherwise + # it will crash + if line[1].strip().isdigit(): + entry.delay = int(line[1]) + state = 2 + elif line[0] == "request-rate": + if state != 0: + numbers = line[1].split('/') + # check if all values are sane + if (len(numbers) == 2 and numbers[0].strip().isdigit() + and numbers[1].strip().isdigit()): + req_rate = collections.namedtuple('req_rate', + 'requests seconds') + entry.req_rate = req_rate + entry.req_rate.requests = int(numbers[0]) + entry.req_rate.seconds = int(numbers[1]) + state = 2 if state == 2: self._add_entry(entry) - def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -153,6 +174,18 @@ class RobotFileParser: # agent not found ==> access granted return True + def crawl_delay(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.delay + return None + + def request_rate(self, useragent): + for entry in self.entries: + if entry.applies_to(useragent): + return entry.req_rate + return None + def __str__(self): return ''.join([str(entry) + "\n" for entry in self.entries]) @@ -180,6 +213,8 @@ class Entry: def __init__(self): self.useragents = [] self.rulelines = [] + self.delay = None + self.req_rate = None def __str__(self): ret = [] diff --git a/Misc/ACKS b/Misc/ACKS index 9678bc3..221d6e2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -151,6 +151,7 @@ Finn Bock Paul Boddie Matthew Boedicker Robin Boerdijk +Nikolay Bogoychev David Bolen Wouter Bolsterlee Gawain Bolton diff --git a/Misc/NEWS b/Misc/NEWS index daa40fc..b543088 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,4 +1,4 @@ -+++++++++++ ++++++++++++ Python News +++++++++++ @@ -46,6 +46,9 @@ Core and Builtins Library ------- +- Issue #16099: RobotFileParser now supports Crawl-delay and Request-rate + extensions. Patch by Nikolay Bogoychev. + - Issue #25316: distutils raises OSError instead of DistutilsPlatformError when MSVC is not installed. -- cgit v0.12 From b6c9572fa9e88029beda2d9cc6b387020f0a7d69 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 8 Oct 2015 13:58:49 +0300 Subject: Sort module names in whatsnew/3.6.rst --- Doc/whatsnew/3.6.rst | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3080820..50d2a6a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -95,12 +95,21 @@ New Modules Improved Modules ================ +datetime +-------- + +:meth:`datetime.stftime ` and +:meth:`date.stftime ` methods now support ISO 8601 date +directives ``%G``, ``%u`` and ``%V``. +(Contributed by Ashley Anderson in :issue:`12006`.) + + operator -------- -* New object :data:`operator.subscript` makes it easier to create complex - indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` - (Contributed by Joe Jevnik in :issue:`24379`.) +New object :data:`operator.subscript` makes it easier to create complex +indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` +(Contributed by Joe Jevnik in :issue:`24379`.) rlcomplete @@ -110,14 +119,6 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) -datetime --------- - -* :meth:`datetime.stftime ` and - :meth:`date.stftime ` methods now support ISO 8601 - date directives ``%G``, ``%u`` and ``%V``. - (Contributed by Ashley Anderson in :issue:`12006`.) - urllib.robotparser ------------------ -- cgit v0.12 From 12c2945ccf8624e3978c5d2069dd18194f88becf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 09:05:36 -0700 Subject: Issue #23919: Prevents assert dialogs appearing in the test suite. --- Lib/test/libregrtest/cmdline.py | 4 ++++ Lib/test/libregrtest/setup.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 6b18b3a..21c3e66 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -304,6 +304,10 @@ def _parse_args(args, **kwargs): if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3): parser.error("--pgo/-v don't go together!") + if ns.nowindows: + print("Warning: the --nowindows (-n) option is deprecated. " + "Use -vv to display assertions in stderr.", file=sys.stderr) + if ns.quiet: ns.verbose = 0 if ns.timeout is not None: diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 6a1c308..6e05c7e 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -75,8 +75,11 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) - if ns.nowindows: + try: import msvcrt + except ImportError: + pass + else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| @@ -88,8 +91,11 @@ def setup_tests(ns): pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + if ns.verbose and ns.verbose >= 2: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) + else: + msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources -- cgit v0.12 From 08ec6d9611b612182808bdf1d30cdd786cde9c3e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 8 Oct 2015 11:34:07 -0700 Subject: Fix missing import in libregrtest. --- Lib/test/libregrtest/cmdline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 21c3e66..a9cee6e 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -1,5 +1,6 @@ import argparse import os +import sys from test import support -- cgit v0.12 From fdfbf781140f22619b0ef6bfeac792496774bb69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 00:33:49 +0200 Subject: Issue #25318: Add _PyBytesWriter API Add a new private API to optimize Unicode encoders. It uses a small buffer allocated on the stack and supports overallocation. Use _PyBytesWriter API for UCS1 (ASCII and Latin1) and UTF-8 encoders. Enable overallocation for the UTF-8 encoder with error handlers. unicode_encode_ucs1(): initialize collend to collstart+1 to not check the current character twice, we already know that it is not ASCII. --- Include/unicodeobject.h | 2 +- Objects/stringlib/codecs.h | 84 +++--------- Objects/unicodeobject.c | 316 +++++++++++++++++++++++++++++++++++---------- 3 files changed, 269 insertions(+), 133 deletions(-) diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index d0e0142..adcb64c 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -908,7 +908,7 @@ typedef struct { /* minimum character (default: 127, ASCII) */ Py_UCS4 min_char; - /* If non-zero, overallocate the buffer by 25% (default: 0). */ + /* If non-zero, overallocate the buffer (default: 0). */ unsigned char overallocate; /* If readonly is 1, buffer is a shared string (cannot be modified) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 562191c..d7a9918 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -263,10 +263,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */ Py_ssize_t i; /* index into s of next input byte */ - PyObject *result; /* result string object */ char *p; /* next free byte in output buffer */ - Py_ssize_t nallocated; /* number of result bytes allocated */ - Py_ssize_t nneeded; /* number of result bytes needed */ #if STRINGLIB_SIZEOF_CHAR > 1 PyObject *error_handler_obj = NULL; PyObject *exc = NULL; @@ -275,39 +272,25 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #endif #if STRINGLIB_SIZEOF_CHAR == 1 const Py_ssize_t max_char_size = 2; - char stackbuf[MAX_SHORT_UNICHARS * 2]; #elif STRINGLIB_SIZEOF_CHAR == 2 const Py_ssize_t max_char_size = 3; - char stackbuf[MAX_SHORT_UNICHARS * 3]; #else /* STRINGLIB_SIZEOF_CHAR == 4 */ const Py_ssize_t max_char_size = 4; - char stackbuf[MAX_SHORT_UNICHARS * 4]; #endif + _PyBytesWriter writer; assert(size >= 0); + _PyBytesWriter_Init(&writer); - if (size <= MAX_SHORT_UNICHARS) { - /* Write into the stack buffer; nallocated can't overflow. - * At the end, we'll allocate exactly as much heap space as it - * turns out we need. - */ - nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int); - result = NULL; /* will allocate after we're done */ - p = stackbuf; - } - else { - if (size > PY_SSIZE_T_MAX / max_char_size) { - /* integer overflow */ - return PyErr_NoMemory(); - } - /* Overallocate on the heap, and give the excess back at the end. */ - nallocated = size * max_char_size; - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - return NULL; - p = PyBytes_AS_STRING(result); + if (size > PY_SSIZE_T_MAX / max_char_size) { + /* integer overflow */ + return PyErr_NoMemory(); } + p = _PyBytesWriter_Alloc(&writer, size * max_char_size); + if (p == NULL) + return NULL; + for (i = 0; i < size;) { Py_UCS4 ch = data[i++]; @@ -338,6 +321,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, while ((endpos < size) && Py_UNICODE_IS_SURROGATE(data[endpos])) endpos++; + /* Only overallocate the buffer if it's not the last write */ + writer.overallocate = (endpos < size); + switch (error_handler) { case _Py_ERROR_REPLACE: @@ -387,29 +373,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, repsize = PyUnicode_GET_LENGTH(rep); if (repsize > max_char_size) { - Py_ssize_t offset; - - if (result == NULL) - offset = p - stackbuf; - else - offset = p - PyBytes_AS_STRING(result); - - if (nallocated > PY_SSIZE_T_MAX - repsize + max_char_size) { - /* integer overflow */ - PyErr_NoMemory(); + p = _PyBytesWriter_Prepare(&writer, p, + repsize - max_char_size); + if (p == NULL) goto error; - } - nallocated += repsize - max_char_size; - if (result != NULL) { - if (_PyBytes_Resize(&result, nallocated) < 0) - goto error; - } else { - result = PyBytes_FromStringAndSize(NULL, nallocated); - if (result == NULL) - goto error; - Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } - p = PyBytes_AS_STRING(result) + offset; } if (PyBytes_Check(rep)) { @@ -437,6 +404,10 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, i = newpos; } + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || i == size); } else #if STRINGLIB_SIZEOF_CHAR > 2 @@ -461,31 +432,18 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #endif /* STRINGLIB_SIZEOF_CHAR > 1 */ } - if (result == NULL) { - /* This was stack allocated. */ - nneeded = p - stackbuf; - assert(nneeded <= nallocated); - result = PyBytes_FromStringAndSize(stackbuf, nneeded); - } - else { - /* Cut back to size actually needed. */ - nneeded = p - PyBytes_AS_STRING(result); - assert(nneeded <= nallocated); - _PyBytes_Resize(&result, nneeded); - } - #if STRINGLIB_SIZEOF_CHAR > 1 Py_XDECREF(error_handler_obj); Py_XDECREF(exc); #endif - return result; + return _PyBytesWriter_Finish(&writer, p); #if STRINGLIB_SIZEOF_CHAR > 1 error: Py_XDECREF(rep); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - Py_XDECREF(result); + _PyBytesWriter_Dealloc(&writer); return NULL; #endif diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 3d78404..010a610 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -163,6 +163,14 @@ extern "C" { *_to++ = (to_type) *_iter++; \ } while (0) +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + /* This dictionary holds all interned unicode strings. Note that references to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation @@ -338,6 +346,216 @@ PyUnicode_GetMax(void) #endif } +/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes object */ + PyObject *buffer; + + /* Number of allocated size */ + Py_ssize_t allocated; + + /* Current size of the buffer (can be smaller than the allocated size) */ + Py_ssize_t size; + + /* If non-zero, overallocate the buffer (default: 0). */ + int overallocate; + + /* Stack buffer */ + int use_stack_buffer; + char stack_buffer[512]; +} _PyBytesWriter; + +static void +_PyBytesWriter_Init(_PyBytesWriter *writer) +{ + writer->buffer = NULL; + writer->allocated = 0; + writer->size = 0; + writer->overallocate = 0; + writer->use_stack_buffer = 0; +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); +#endif +} + +static void +_PyBytesWriter_Dealloc(_PyBytesWriter *writer) +{ + Py_CLEAR(writer->buffer); +} + +static char* +_PyBytesWriter_AsString(_PyBytesWriter *writer) +{ + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); + } + else { + assert(writer->buffer == NULL); + return writer->stack_buffer; + } +} + +Py_LOCAL_INLINE(Py_ssize_t) +_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +{ + char *start = _PyBytesWriter_AsString(writer); + assert(str != NULL); + assert(str >= start); + return str - start; +} + +Py_LOCAL_INLINE(void) +_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) +{ +#ifdef Py_DEBUG + char *start, *end; + + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + assert(PyBytes_CheckExact(writer->buffer)); + assert(Py_REFCNT(writer->buffer) == 1); + } + else { + assert(writer->buffer == NULL); + } + + start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->size && writer->size <= writer->allocated); + /* the last byte must always be null */ + assert(start[writer->allocated] == 0); + + end = start + writer->allocated; + assert(str != NULL); + assert(start <= str && str <= end); +#endif +} + +/* Add *size* bytes to the buffer. + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +static char* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +{ + Py_ssize_t allocated, pos; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + writer->size += size; + + allocated = writer->allocated; + if (writer->size <= allocated) + return str; + + allocated = writer->size; + if (writer->overallocate + && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + allocated += allocated / OVERALLOCATE_FACTOR; + } + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + /* Note: Don't use a bytearray object because the conversion from + byterray to bytes requires to copy all bytes. */ + if (_PyBytes_Resize(&writer->buffer, allocated)) { + assert(writer->buffer == NULL); + return NULL; + } + } + else { + /* convert from stack buffer to bytes object buffer */ + assert(writer->buffer == NULL); + + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->buffer == NULL) + return NULL; + + if (pos != 0) { + Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + writer->stack_buffer, + pos); + } + +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); +#endif + + writer->use_stack_buffer = 0; + } + writer->allocated = allocated; + + str = _PyBytesWriter_AsString(writer) + pos; + _PyBytesWriter_CheckConsistency(writer, str); + return str; +} + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +static char* +_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) +{ + /* ensure that _PyBytesWriter_Alloc() is only called once */ + assert(writer->size == 0 && writer->buffer == NULL); + assert(size >= 0); + + writer->use_stack_buffer = 1; +#if Py_DEBUG + /* the last byte is reserved, it must be '\0' */ + writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; + writer->allocated = sizeof(writer->stack_buffer) - 1; +#else + writer->allocated = sizeof(writer->stack_buffer); +#endif + return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); +} + +/* Get the buffer content and reset the writer. + Return a bytes object. + Raise an exception and return NULL on error. */ +static PyObject * +_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +{ + Py_ssize_t pos; + PyObject *result; + + _PyBytesWriter_CheckConsistency(writer, str); + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + if (pos != writer->allocated) { + if (_PyBytes_Resize(&writer->buffer, pos)) { + assert(writer->buffer == NULL); + return NULL; + } + } + + result = writer->buffer; + writer->buffer = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + } + + return result; +} + #ifdef Py_DEBUG int _PyUnicode_CheckConsistency(PyObject *op, int check_content) @@ -6460,17 +6678,15 @@ unicode_encode_ucs1(PyObject *unicode, Py_ssize_t pos=0, size; int kind; void *data; - /* output object */ - PyObject *res; /* pointer into the output */ char *str; - /* current output position */ - Py_ssize_t ressize; const char *encoding = (limit == 256) ? "latin-1" : "ascii"; const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)"; PyObject *error_handler_obj = NULL; PyObject *exc = NULL; _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + /* output object */ + _PyBytesWriter writer; if (PyUnicode_READY(unicode) == -1) return NULL; @@ -6481,11 +6697,11 @@ unicode_encode_ucs1(PyObject *unicode, replacements, if we need more, we'll resize */ if (size == 0) return PyBytes_FromStringAndSize(NULL, 0); - res = PyBytes_FromStringAndSize(NULL, size); - if (res == NULL) + + _PyBytesWriter_Init(&writer); + str = _PyBytesWriter_Alloc(&writer, size); + if (str == NULL) return NULL; - str = PyBytes_AS_STRING(res); - ressize = size; while (pos < size) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); @@ -6499,15 +6715,18 @@ unicode_encode_ucs1(PyObject *unicode, else { Py_ssize_t requiredsize; PyObject *repunicode; - Py_ssize_t repsize, newpos, respos, i; + Py_ssize_t repsize, newpos, i; /* startpos for collecting unencodable chars */ Py_ssize_t collstart = pos; - Py_ssize_t collend = pos; + Py_ssize_t collend = collstart + 1; /* find all unecodable characters */ while ((collend < size) && (PyUnicode_READ(kind, data, collend) >= limit)) ++collend; + /* Only overallocate the buffer if it's not the last write */ + writer.overallocate = (collend < size); + /* cache callback name lookup (if not done yet, i.e. it's the first error) */ if (error_handler == _Py_ERROR_UNKNOWN) error_handler = get_error_handler(errors); @@ -6526,8 +6745,7 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - respos = str - PyBytes_AS_STRING(res); - requiredsize = respos; + requiredsize = 0; /* determine replacement size */ for (i = collstart; i < collend; ++i) { Py_ssize_t incr; @@ -6553,17 +6771,11 @@ unicode_encode_ucs1(PyObject *unicode, goto overflow; requiredsize += incr; } - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) - goto onError; - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; - } + + str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1); + if (str == NULL) + goto onError; + /* generate replacement */ for (i = collstart; i < collend; ++i) { str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); @@ -6598,20 +6810,9 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - if (repsize > 1) { - /* Make room for all additional bytes. */ - respos = str - PyBytes_AS_STRING(res); - if (ressize > PY_SSIZE_T_MAX - repsize - 1) { - Py_DECREF(repunicode); - goto overflow; - } - if (_PyBytes_Resize(&res, ressize+repsize-1)) { - Py_DECREF(repunicode); - goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize += repsize-1; - } + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) + goto onError; memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6622,24 +6823,11 @@ unicode_encode_ucs1(PyObject *unicode, /* need more space? (at least enough for what we have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ - respos = str - PyBytes_AS_STRING(res); repsize = PyUnicode_GET_LENGTH(repunicode); - requiredsize = respos; - if (requiredsize > PY_SSIZE_T_MAX - repsize) - goto overflow; - requiredsize += repsize; - if (requiredsize > PY_SSIZE_T_MAX - (size - collend)) - goto overflow; - requiredsize += size - collend; - if (requiredsize > ressize) { - if (ressize <= PY_SSIZE_T_MAX/2 && requiredsize < 2*ressize) - requiredsize = 2*ressize; - if (_PyBytes_Resize(&res, requiredsize)) { - Py_DECREF(repunicode); + if (repsize > 1) { + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) goto onError; - } - str = PyBytes_AS_STRING(res) + respos; - ressize = requiredsize; } /* check if there is anything unencodable in the replacement @@ -6657,26 +6845,23 @@ unicode_encode_ucs1(PyObject *unicode, pos = newpos; Py_DECREF(repunicode); } + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || pos == size); } } - /* Resize if we allocated to much */ - size = str - PyBytes_AS_STRING(res); - if (size < ressize) { /* If this falls res will be NULL */ - assert(size >= 0); - if (_PyBytes_Resize(&res, size) < 0) - goto onError; - } Py_XDECREF(error_handler_obj); Py_XDECREF(exc); - return res; + return _PyBytesWriter_Finish(&writer, str); overflow: PyErr_SetString(PyExc_OverflowError, "encoded result is too long for a Python string"); onError: - Py_XDECREF(res); + _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); return NULL; @@ -13366,13 +13551,6 @@ int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) { -#ifdef MS_WINDOWS - /* On Windows, overallocate by 50% is the best factor */ -# define OVERALLOCATE_FACTOR 2 -#else - /* On Linux, overallocate by 25% is the best factor */ -# define OVERALLOCATE_FACTOR 4 -#endif Py_ssize_t newlen; PyObject *newbuffer; -- cgit v0.12 From e7bf86cd7d7c9a3924501875a08c4ef4a0063103 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 01:39:28 +0200 Subject: Optimize backslashreplace error handler Issue #25318: Optimize backslashreplace and xmlcharrefreplace error handlers in UTF-8 encoder. Optimize also backslashreplace error handler for ASCII and Latin1 encoders. Use the new _PyBytesWriter API to optimize these error handlers for the encoders. It avoids to create an exception and call the slow implementation of the error handler. --- Objects/stringlib/codecs.h | 18 ++++- Objects/unicodeobject.c | 193 +++++++++++++++++++++++++++++++++------------ 2 files changed, 160 insertions(+), 51 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index d7a9918..ae99d1a 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -334,7 +334,6 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, i += (endpos - startpos - 1); break; - case _Py_ERROR_SURROGATEPASS: for (k=startpos; k PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + prealloc = prealloc_per_char * (collend - collstart); + if (size > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, size - prealloc); + if (str == NULL) + return NULL; + } + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + ch = PyUnicode_READ(kind, data, i); + if (ch < 0x100) + str += sprintf(str, "\\x%02x", ch); + else if (ch < 0x10000) + str += sprintf(str, "\\u%04x", ch); + else { + assert(ch <= MAX_UNICODE); + str += sprintf(str, "\\U%08x", ch); + } + } + return str; +} + +/* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: + ASCII, Latin1, UTF-8, etc. */ +static char* +xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, + char *str, + PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) +{ + Py_ssize_t size, i, prealloc; + Py_UCS4 ch; + enum PyUnicode_Kind kind; + void *data; + + assert(PyUnicode_IS_READY(unicode)); + kind = PyUnicode_KIND(unicode); + data = PyUnicode_DATA(unicode); + + size = 0; + /* determine replacement size */ + for (i = collstart; i < collend; ++i) { + Py_ssize_t incr; + + ch = PyUnicode_READ(kind, data, i); + if (ch < 10) + incr = 2+1+1; + else if (ch < 100) + incr = 2+2+1; + else if (ch < 1000) + incr = 2+3+1; + else if (ch < 10000) + incr = 2+4+1; + else if (ch < 100000) + incr = 2+5+1; + else if (ch < 1000000) + incr = 2+6+1; + else { + assert(ch <= MAX_UNICODE); + incr = 2+7+1; + } + if (size > PY_SSIZE_T_MAX - incr) { + PyErr_SetString(PyExc_OverflowError, + "encoded result is too long for a Python string"); + return NULL; + } + size += incr; + } + + prealloc = prealloc_per_char * (collend - collstart); + if (size > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, size - prealloc); + if (str == NULL) + return NULL; + } + + /* generate replacement */ + for (i = collstart; i < collend; ++i) { + str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); + } + return str; +} + /* --- Bloom Filters ----------------------------------------------------- */ /* stuff to implement simple "bloom filters" for Unicode characters. @@ -6713,7 +6834,6 @@ unicode_encode_ucs1(PyObject *unicode, ++pos; } else { - Py_ssize_t requiredsize; PyObject *repunicode; Py_ssize_t repsize, newpos, i; /* startpos for collecting unencodable chars */ @@ -6744,42 +6864,19 @@ unicode_encode_ucs1(PyObject *unicode, pos = collend; break; - case _Py_ERROR_XMLCHARREFREPLACE: - requiredsize = 0; - /* determine replacement size */ - for (i = collstart; i < collend; ++i) { - Py_ssize_t incr; - - ch = PyUnicode_READ(kind, data, i); - if (ch < 10) - incr = 2+1+1; - else if (ch < 100) - incr = 2+2+1; - else if (ch < 1000) - incr = 2+3+1; - else if (ch < 10000) - incr = 2+4+1; - else if (ch < 100000) - incr = 2+5+1; - else if (ch < 1000000) - incr = 2+6+1; - else { - assert(ch <= MAX_UNICODE); - incr = 2+7+1; - } - if (requiredsize > PY_SSIZE_T_MAX - incr) - goto overflow; - requiredsize += incr; - } - - str = _PyBytesWriter_Prepare(&writer, str, requiredsize-1); + case _Py_ERROR_BACKSLASHREPLACE: + str = backslashreplace(&writer, 1, str, + unicode, collstart, collend); if (str == NULL) goto onError; + pos = collend; + break; - /* generate replacement */ - for (i = collstart; i < collend; ++i) { - str += sprintf(str, "&#%d;", PyUnicode_READ(kind, data, i)); - } + case _Py_ERROR_XMLCHARREFREPLACE: + str = xmlcharrefreplace(&writer, 1, str, + unicode, collstart, collend); + if (str == NULL) + goto onError; pos = collend; break; @@ -6810,9 +6907,11 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; + if (repsize > 1) { + str = _PyBytesWriter_Prepare(&writer, str, repsize-1); + if (str == NULL) + goto onError; + } memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6856,10 +6955,6 @@ unicode_encode_ucs1(PyObject *unicode, Py_XDECREF(exc); return _PyBytesWriter_Finish(&writer, str); - overflow: - PyErr_SetString(PyExc_OverflowError, - "encoded result is too long for a Python string"); - onError: _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); -- cgit v0.12 From 0016507c168fa942d7856bdef371cd8d494b140b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 01:53:21 +0200 Subject: Issue #25318: Move _PyBytesWriter to bytesobject.c Declare also the private API in bytesobject.h. --- Include/bytesobject.h | 52 ++++++++++++ Objects/bytesobject.c | 193 ++++++++++++++++++++++++++++++++++++++++++++ Objects/unicodeobject.c | 210 ------------------------------------------------ 3 files changed, 245 insertions(+), 210 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index e379bac..eafcdea 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -123,6 +123,58 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, #define F_ALT (1<<3) #define F_ZERO (1<<4) +#ifndef Py_LIMITED_API +/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes object */ + PyObject *buffer; + + /* Number of allocated size */ + Py_ssize_t allocated; + + /* Current size of the buffer (can be smaller than the allocated size) */ + Py_ssize_t size; + + /* If non-zero, overallocate the buffer (default: 0). */ + int overallocate; + + /* Stack buffer */ + int use_stack_buffer; + char stack_buffer[512]; +} _PyBytesWriter; + +/* Initialize a bytes writer + + By default, the overallocation is disabled. Set the overallocate attribute + to control the allocation of the buffer. */ +PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); + +/* Get the buffer content and reset the writer. + Return a bytes object. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, + char *str); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, + Py_ssize_t size); + +/* Add *size* bytes to the buffer. + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + char *str, + Py_ssize_t size); +#endif /* Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 08275ad..46322aa 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3735,3 +3735,196 @@ bytes_iter(PyObject *seq) _PyObject_GC_TRACK(it); return (PyObject *)it; } + + +/* _PyBytesWriter API */ + +#ifdef MS_WINDOWS + /* On Windows, overallocate by 50% is the best factor */ +# define OVERALLOCATE_FACTOR 2 +#else + /* On Linux, overallocate by 25% is the best factor */ +# define OVERALLOCATE_FACTOR 4 +#endif + +void +_PyBytesWriter_Init(_PyBytesWriter *writer) +{ + writer->buffer = NULL; + writer->allocated = 0; + writer->size = 0; + writer->overallocate = 0; + writer->use_stack_buffer = 0; +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); +#endif +} + +void +_PyBytesWriter_Dealloc(_PyBytesWriter *writer) +{ + Py_CLEAR(writer->buffer); +} + +Py_LOCAL_INLINE(char*) +_PyBytesWriter_AsString(_PyBytesWriter *writer) +{ + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); + } + else { + assert(writer->buffer == NULL); + return writer->stack_buffer; + } +} + +Py_LOCAL_INLINE(Py_ssize_t) +_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +{ + char *start = _PyBytesWriter_AsString(writer); + assert(str != NULL); + assert(str >= start); + return str - start; +} + +Py_LOCAL_INLINE(void) +_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) +{ +#ifdef Py_DEBUG + char *start, *end; + + if (!writer->use_stack_buffer) { + assert(writer->buffer != NULL); + assert(PyBytes_CheckExact(writer->buffer)); + assert(Py_REFCNT(writer->buffer) == 1); + } + else { + assert(writer->buffer == NULL); + } + + start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->size && writer->size <= writer->allocated); + /* the last byte must always be null */ + assert(start[writer->allocated] == 0); + + end = start + writer->allocated; + assert(str != NULL); + assert(start <= str && str <= end); +#endif +} + +char* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +{ + Py_ssize_t allocated, pos; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + writer->size += size; + + allocated = writer->allocated; + if (writer->size <= allocated) + return str; + + allocated = writer->size; + if (writer->overallocate + && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { + /* overallocate to limit the number of realloc() */ + allocated += allocated / OVERALLOCATE_FACTOR; + } + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + /* Note: Don't use a bytearray object because the conversion from + byterray to bytes requires to copy all bytes. */ + if (_PyBytes_Resize(&writer->buffer, allocated)) { + assert(writer->buffer == NULL); + return NULL; + } + } + else { + /* convert from stack buffer to bytes object buffer */ + assert(writer->buffer == NULL); + + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->buffer == NULL) + return NULL; + + if (pos != 0) { + Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + writer->stack_buffer, + pos); + } + +#ifdef Py_DEBUG + memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); +#endif + + writer->use_stack_buffer = 0; + } + writer->allocated = allocated; + + str = _PyBytesWriter_AsString(writer) + pos; + _PyBytesWriter_CheckConsistency(writer, str); + return str; +} + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +char* +_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) +{ + /* ensure that _PyBytesWriter_Alloc() is only called once */ + assert(writer->size == 0 && writer->buffer == NULL); + assert(size >= 0); + + writer->use_stack_buffer = 1; +#if Py_DEBUG + /* the last byte is reserved, it must be '\0' */ + writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; + writer->allocated = sizeof(writer->stack_buffer) - 1; +#else + writer->allocated = sizeof(writer->stack_buffer); +#endif + return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); +} + +PyObject * +_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +{ + Py_ssize_t pos; + PyObject *result; + + _PyBytesWriter_CheckConsistency(writer, str); + + pos = _PyBytesWriter_GetPos(writer, str); + if (!writer->use_stack_buffer) { + if (pos != writer->allocated) { + if (_PyBytes_Resize(&writer->buffer, pos)) { + assert(writer->buffer == NULL); + return NULL; + } + } + + result = writer->buffer; + writer->buffer = NULL; + } + else { + result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + } + + return result; +} diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index e0b3c68..614d214 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -347,216 +347,6 @@ PyUnicode_GetMax(void) #endif } -/* The _PyBytesWriter structure is big: it contains an embeded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes object */ - PyObject *buffer; - - /* Number of allocated size */ - Py_ssize_t allocated; - - /* Current size of the buffer (can be smaller than the allocated size) */ - Py_ssize_t size; - - /* If non-zero, overallocate the buffer (default: 0). */ - int overallocate; - - /* Stack buffer */ - int use_stack_buffer; - char stack_buffer[512]; -} _PyBytesWriter; - -static void -_PyBytesWriter_Init(_PyBytesWriter *writer) -{ - writer->buffer = NULL; - writer->allocated = 0; - writer->size = 0; - writer->overallocate = 0; - writer->use_stack_buffer = 0; -#ifdef Py_DEBUG - memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); -#endif -} - -static void -_PyBytesWriter_Dealloc(_PyBytesWriter *writer) -{ - Py_CLEAR(writer->buffer); -} - -static char* -_PyBytesWriter_AsString(_PyBytesWriter *writer) -{ - if (!writer->use_stack_buffer) { - assert(writer->buffer != NULL); - return PyBytes_AS_STRING(writer->buffer); - } - else { - assert(writer->buffer == NULL); - return writer->stack_buffer; - } -} - -Py_LOCAL_INLINE(Py_ssize_t) -_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) -{ - char *start = _PyBytesWriter_AsString(writer); - assert(str != NULL); - assert(str >= start); - return str - start; -} - -Py_LOCAL_INLINE(void) -_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) -{ -#ifdef Py_DEBUG - char *start, *end; - - if (!writer->use_stack_buffer) { - assert(writer->buffer != NULL); - assert(PyBytes_CheckExact(writer->buffer)); - assert(Py_REFCNT(writer->buffer) == 1); - } - else { - assert(writer->buffer == NULL); - } - - start = _PyBytesWriter_AsString(writer); - assert(0 <= writer->size && writer->size <= writer->allocated); - /* the last byte must always be null */ - assert(start[writer->allocated] == 0); - - end = start + writer->allocated; - assert(str != NULL); - assert(start <= str && str <= end); -#endif -} - -/* Add *size* bytes to the buffer. - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -static char* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) -{ - Py_ssize_t allocated, pos; - - _PyBytesWriter_CheckConsistency(writer, str); - assert(size >= 0); - - if (size == 0) { - /* nothing to do */ - return str; - } - - if (writer->size > PY_SSIZE_T_MAX - size) { - PyErr_NoMemory(); - _PyBytesWriter_Dealloc(writer); - return NULL; - } - writer->size += size; - - allocated = writer->allocated; - if (writer->size <= allocated) - return str; - - allocated = writer->size; - if (writer->overallocate - && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - allocated += allocated / OVERALLOCATE_FACTOR; - } - - pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { - /* Note: Don't use a bytearray object because the conversion from - byterray to bytes requires to copy all bytes. */ - if (_PyBytes_Resize(&writer->buffer, allocated)) { - assert(writer->buffer == NULL); - return NULL; - } - } - else { - /* convert from stack buffer to bytes object buffer */ - assert(writer->buffer == NULL); - - writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); - if (writer->buffer == NULL) - return NULL; - - if (pos != 0) { - Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), - writer->stack_buffer, - pos); - } - -#ifdef Py_DEBUG - memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); -#endif - - writer->use_stack_buffer = 0; - } - writer->allocated = allocated; - - str = _PyBytesWriter_AsString(writer) + pos; - _PyBytesWriter_CheckConsistency(writer, str); - return str; -} - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -static char* -_PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) -{ - /* ensure that _PyBytesWriter_Alloc() is only called once */ - assert(writer->size == 0 && writer->buffer == NULL); - assert(size >= 0); - - writer->use_stack_buffer = 1; -#if Py_DEBUG - /* the last byte is reserved, it must be '\0' */ - writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; - writer->allocated = sizeof(writer->stack_buffer) - 1; -#else - writer->allocated = sizeof(writer->stack_buffer); -#endif - return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); -} - -/* Get the buffer content and reset the writer. - Return a bytes object. - Raise an exception and return NULL on error. */ -static PyObject * -_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) -{ - Py_ssize_t pos; - PyObject *result; - - _PyBytesWriter_CheckConsistency(writer, str); - - pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { - if (pos != writer->allocated) { - if (_PyBytes_Resize(&writer->buffer, pos)) { - assert(writer->buffer == NULL); - return NULL; - } - } - - result = writer->buffer; - writer->buffer = NULL; - } - else { - result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); - } - - return result; -} - #ifdef Py_DEBUG int _PyUnicode_CheckConsistency(PyObject *op, int check_content) -- cgit v0.12 From b13b97d3b881daaab8f3bf32cc851eac9b895c52 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 02:52:16 +0200 Subject: Issue #25318: Fix compilation error Replace "#if Py_DEBUG" with "#ifdef Py_DEBUG". --- Objects/bytesobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 46322aa..56231a2 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3892,7 +3892,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) assert(size >= 0); writer->use_stack_buffer = 1; -#if Py_DEBUG +#ifdef Py_DEBUG /* the last byte is reserved, it must be '\0' */ writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; writer->allocated = sizeof(writer->stack_buffer) - 1; -- cgit v0.12 From 797485e10135ca323565b22b4fabf1e161a5ec7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:17:30 +0200 Subject: Issue #25318: Avoid sprintf() in backslashreplace() Rewrite backslashreplace() to be closer to PyCodec_BackslashReplaceErrors(). Add also unit tests for non-BMP characters. --- Lib/test/test_codecs.py | 6 ++++-- Objects/unicodeobject.c | 25 ++++++++++++++++++------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 7b6883f..ff314b1 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3155,7 +3155,8 @@ class ASCIITest(unittest.TestCase): ('[\x80\xff\u20ac]', 'ignore', b'[]'), ('[\x80\xff\u20ac]', 'replace', b'[???]'), ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[€ÿ€]'), - ('[\x80\xff\u20ac]', 'backslashreplace', b'[\\x80\\xff\\u20ac]'), + ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace', + b'[\\x80\\xff\\u20ac\\U000abcde]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): with self.subTest(data=data, error_handler=error_handler, @@ -3197,7 +3198,8 @@ class Latin1Test(unittest.TestCase): for data, error_handler, expected in ( ('[\u20ac\udc80]', 'ignore', b'[]'), ('[\u20ac\udc80]', 'replace', b'[??]'), - ('[\u20ac\udc80]', 'backslashreplace', b'[\\u20ac\\udc80]'), + ('[\u20ac\U000abcde]', 'backslashreplace', + b'[\\u20ac\\U000abcde]'), ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[€�]'), ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'), ): diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 614d214..10cdcc0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -610,14 +610,25 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, /* generate replacement */ for (i = collstart; i < collend; ++i) { ch = PyUnicode_READ(kind, data, i); - if (ch < 0x100) - str += sprintf(str, "\\x%02x", ch); - else if (ch < 0x10000) - str += sprintf(str, "\\u%04x", ch); - else { - assert(ch <= MAX_UNICODE); - str += sprintf(str, "\\U%08x", ch); + *str++ = '\\'; + if (ch >= 0x00010000) { + *str++ = 'U'; + *str++ = Py_hexdigits[(ch>>28)&0xf]; + *str++ = Py_hexdigits[(ch>>24)&0xf]; + *str++ = Py_hexdigits[(ch>>20)&0xf]; + *str++ = Py_hexdigits[(ch>>16)&0xf]; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; + } + else if (ch >= 0x100) { + *str++ = 'u'; + *str++ = Py_hexdigits[(ch>>12)&0xf]; + *str++ = Py_hexdigits[(ch>>8)&0xf]; } + else + *str++ = 'x'; + *str++ = Py_hexdigits[(ch>>4)&0xf]; + *str++ = Py_hexdigits[ch&0xf]; } return str; } -- cgit v0.12 From 3fa36ff5e4991550e31cc7ab55dc3a2165c2ffa3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:37:11 +0200 Subject: Issue #25318: Fix backslashreplace() Fix code to estimate the needed space. --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 10cdcc0..a3bbf92 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -590,7 +590,7 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, incr = 2+4; else { assert(ch <= MAX_UNICODE); - incr = 2+6; + incr = 2+8; } if (size > PY_SSIZE_T_MAX - incr) { PyErr_SetString(PyExc_OverflowError, -- cgit v0.12 From b3653a34580684d210daed5cbb8a676c4d7a33fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 03:38:24 +0200 Subject: Issue #25318: cleanup code _PyBytesWriter Rename "stack buffer" to "small buffer". Add also an assertion in _PyBytesWriter_GetPos(). --- Include/bytesobject.h | 4 ++-- Objects/bytesobject.c | 34 +++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index eafcdea..ffa529b 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -141,8 +141,8 @@ typedef struct { int overallocate; /* Stack buffer */ - int use_stack_buffer; - char stack_buffer[512]; + int use_small_buffer; + char small_buffer[512]; } _PyBytesWriter; /* Initialize a bytes writer diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 56231a2..722978d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3754,9 +3754,9 @@ _PyBytesWriter_Init(_PyBytesWriter *writer) writer->allocated = 0; writer->size = 0; writer->overallocate = 0; - writer->use_stack_buffer = 0; + writer->use_small_buffer = 0; #ifdef Py_DEBUG - memset(writer->stack_buffer, 0xCB, sizeof(writer->stack_buffer)); + memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer)); #endif } @@ -3769,13 +3769,13 @@ _PyBytesWriter_Dealloc(_PyBytesWriter *writer) Py_LOCAL_INLINE(char*) _PyBytesWriter_AsString(_PyBytesWriter *writer) { - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { assert(writer->buffer != NULL); return PyBytes_AS_STRING(writer->buffer); } else { assert(writer->buffer == NULL); - return writer->stack_buffer; + return writer->small_buffer; } } @@ -3785,6 +3785,7 @@ _PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) char *start = _PyBytesWriter_AsString(writer); assert(str != NULL); assert(str >= start); + assert(str - start <= writer->allocated); return str - start; } @@ -3794,7 +3795,7 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #ifdef Py_DEBUG char *start, *end; - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { assert(writer->buffer != NULL); assert(PyBytes_CheckExact(writer->buffer)); assert(Py_REFCNT(writer->buffer) == 1); @@ -3846,7 +3847,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) } pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { /* Note: Don't use a bytearray object because the conversion from byterray to bytes requires to copy all bytes. */ if (_PyBytes_Resize(&writer->buffer, allocated)) { @@ -3864,15 +3865,14 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) if (pos != 0) { Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), - writer->stack_buffer, + writer->small_buffer, pos); } + writer->use_small_buffer = 0; #ifdef Py_DEBUG - memset(writer->stack_buffer, 0xDB, sizeof(writer->stack_buffer)); + memset(writer->small_buffer, 0xDB, sizeof(writer->small_buffer)); #endif - - writer->use_stack_buffer = 0; } writer->allocated = allocated; @@ -3891,15 +3891,15 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) assert(writer->size == 0 && writer->buffer == NULL); assert(size >= 0); - writer->use_stack_buffer = 1; + writer->use_small_buffer = 1; #ifdef Py_DEBUG /* the last byte is reserved, it must be '\0' */ - writer->stack_buffer[sizeof(writer->stack_buffer) - 1] = 0; - writer->allocated = sizeof(writer->stack_buffer) - 1; + writer->allocated = sizeof(writer->small_buffer) - 1; + writer->small_buffer[writer->allocated] = 0; #else - writer->allocated = sizeof(writer->stack_buffer); + writer->allocated = sizeof(writer->small_buffer); #endif - return _PyBytesWriter_Prepare(writer, writer->stack_buffer, size); + return _PyBytesWriter_Prepare(writer, writer->small_buffer, size); } PyObject * @@ -3911,7 +3911,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_stack_buffer) { + if (!writer->use_small_buffer) { if (pos != writer->allocated) { if (_PyBytes_Resize(&writer->buffer, pos)) { assert(writer->buffer == NULL); @@ -3923,7 +3923,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) writer->buffer = NULL; } else { - result = PyBytes_FromStringAndSize(writer->stack_buffer, pos); + result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } return result; -- cgit v0.12 From 7836a27ceba2395572f1b3b385c11e6a43910185 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 00:03:51 -0400 Subject: Issue #25298: Add lock and rlock weakref tests (Contributed by Nir Soffer). --- Lib/test/lock_tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index afd6873..055bf28 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -7,6 +7,7 @@ import time from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest +import weakref from test import support @@ -198,6 +199,17 @@ class BaseLockTests(BaseTestCase): self.assertFalse(results[0]) self.assertTimeout(results[1], 0.5) + def test_weakref_exists(self): + lock = self.locktype() + ref = weakref.ref(lock) + self.assertIsNotNone(ref()) + + def test_weakref_deleted(self): + lock = self.locktype() + ref = weakref.ref(lock) + del lock + self.assertIsNone(ref()) + class LockTests(BaseLockTests): """ -- cgit v0.12 From 5098b58381e572ccd55982004da903a59ffec95c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 00:42:47 -0400 Subject: Make comparison more consistent --- Python/bltinmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 2038d2b..3f2e2c0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -331,7 +331,7 @@ builtin_any(PyModuleDef *module, PyObject *iterable) Py_DECREF(it); return NULL; } - if (cmp == 1) { + if (cmp > 0) { Py_DECREF(it); Py_RETURN_TRUE; } -- cgit v0.12 From bd5f0e8c1cf633a238c10e1d3199c0d572a4df1d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 9 Oct 2015 01:34:08 -0400 Subject: Hoist constant expression out of the inner loop. --- Python/bltinmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3f2e2c0..3c8f1e9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -469,6 +469,7 @@ filter_next(filterobject *lz) PyObject *it = lz->it; long ok; PyObject *(*iternext)(PyObject *); + int checktrue = lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type; iternext = *Py_TYPE(it)->tp_iternext; for (;;) { @@ -476,12 +477,11 @@ filter_next(filterobject *lz) if (item == NULL) return NULL; - if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) { + if (checktrue) { ok = PyObject_IsTrue(item); } else { PyObject *good; - good = PyObject_CallFunctionObjArgs(lz->func, - item, NULL); + good = PyObject_CallFunctionObjArgs(lz->func, item, NULL); if (good == NULL) { Py_DECREF(item); return NULL; -- cgit v0.12 From fa7762ec066aa3632a25b6a52bb7597b8f17c2f3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 11:48:06 +0200 Subject: Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API * Thanks to the _PyBytesWriter API, output smaller than 512 bytes are allocated on the stack and so avoid calling _PyBytes_Resize(). Because of that, change the default buffer size to fmtcnt instead of fmtcnt+100. * Rely on _PyBytesWriter algorithm to overallocate the buffer instead of using a custom code. For example, _PyBytesWriter uses a different overallocation factor (25% or 50%) depending on the platform to get best performances. * Disable overallocation for the last write. * Replace C loops to fill characters with memset() * Add also many comments to _PyBytes_Format() * Remove unused FORMATBUFLEN constant * Avoid the creation of a temporary bytes object when formatting a floating point number (when no custom formatting option is used) * Fix also reference leaks on error handling * Use Py_MEMCPY() to copy bytes between two formatters (%) --- Misc/NEWS | 2 + Objects/bytesobject.c | 187 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 130 insertions(+), 59 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 2dc4d9b..9a7a85a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API. + - Issue #24806: Prevent builtin types that are not allowed to be subclassed from being subclassed through multiple inheritance. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 722978d..ef53288 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -409,12 +409,15 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) /* Returns a new reference to a PyBytes object, or NULL on failure. */ -static PyObject * -formatfloat(PyObject *v, int flags, int prec, int type) +static char* +formatfloat(PyObject *v, int flags, int prec, int type, + PyObject **p_result, _PyBytesWriter *writer, char *str, + Py_ssize_t prealloc) { char *p; PyObject *result; double x; + size_t len; x = PyFloat_AsDouble(v); if (x == -1.0 && PyErr_Occurred()) { @@ -431,9 +434,23 @@ formatfloat(PyObject *v, int flags, int prec, int type) if (p == NULL) return NULL; - result = PyBytes_FromStringAndSize(p, strlen(p)); + + len = strlen(p); + if (writer != NULL) { + if ((Py_ssize_t)len > prealloc) { + str = _PyBytesWriter_Prepare(writer, str, len - prealloc); + if (str == NULL) + return NULL; + } + Py_MEMCPY(str, p, len); + str += len; + return str; + } + + result = PyBytes_FromStringAndSize(p, len); PyMem_Free(p); - return result; + *p_result = result; + return str; } static PyObject * @@ -557,36 +574,32 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) return NULL; } -/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) - - FORMATBUFLEN is the length of the buffer in which the ints & - chars are formatted. XXX This is a magic number. Each formatting - routine does bounds checking to ensure no overflow, but a better - solution may be to malloc a buffer of appropriate size for each - format. For now, the current solution is sufficient. -*/ -#define FORMATBUFLEN (size_t)120 +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * _PyBytes_Format(PyObject *format, PyObject *args) { char *fmt, *res; Py_ssize_t arglen, argidx; - Py_ssize_t reslen, rescnt, fmtcnt; + Py_ssize_t fmtcnt; int args_owned = 0; - PyObject *result; PyObject *dict = NULL; + _PyBytesWriter writer; + if (format == NULL || !PyBytes_Check(format) || args == NULL) { PyErr_BadInternalCall(); return NULL; } fmt = PyBytes_AS_STRING(format); fmtcnt = PyBytes_GET_SIZE(format); - reslen = rescnt = fmtcnt + 100; - result = PyBytes_FromStringAndSize((char *)NULL, reslen); - if (result == NULL) + + _PyBytesWriter_Init(&writer); + + res = _PyBytesWriter_Alloc(&writer, fmtcnt); + if (res == NULL) return NULL; - res = PyBytes_AsString(result); + writer.overallocate = 1; + if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); argidx = 0; @@ -600,18 +613,25 @@ _PyBytes_Format(PyObject *format, PyObject *args) !PyByteArray_Check(args)) { dict = args; } + while (--fmtcnt >= 0) { if (*fmt != '%') { - if (--rescnt < 0) { - rescnt = fmtcnt + 100; - reslen += rescnt; - if (_PyBytes_Resize(&result, reslen)) - return NULL; - res = PyBytes_AS_STRING(result) - + reslen - rescnt; - --rescnt; + Py_ssize_t len; + char *pos; + + pos = strchr(fmt + 1, '%'); + if (pos != NULL) + len = pos - fmt; + else { + len = PyBytes_GET_SIZE(format); + len -= (fmt - PyBytes_AS_STRING(format)); } - *res++ = *fmt++; + assert(len != 0); + + Py_MEMCPY(res, fmt, len); + res += len; + fmt += len; + fmtcnt -= (len - 1); } else { /* Got a format specifier */ @@ -626,6 +646,10 @@ _PyBytes_Format(PyObject *format, PyObject *args) int sign; Py_ssize_t len = 0; char onechar; /* For byte_converter() */ + Py_ssize_t alloc; +#ifdef Py_DEBUG + char *before; +#endif fmt++; if (*fmt == '(') { @@ -673,6 +697,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) arglen = -1; argidx = -2; } + + /* Parse flags. Example: "%+i" => flags=F_SIGN. */ while (--fmtcnt >= 0) { switch (c = *fmt++) { case '-': flags |= F_LJUST; continue; @@ -683,6 +709,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) } break; } + + /* Parse width. Example: "%10s" => width=10 */ if (c == '*') { v = getnextarg(args, arglen, &argidx); if (v == NULL) @@ -717,6 +745,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) width = width*10 + (c - '0'); } } + + /* Parse precision. Example: "%.3f" => prec=3 */ if (c == '.') { prec = 0; if (--fmtcnt >= 0) @@ -771,6 +801,12 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (v == NULL) goto error; } + + if (fmtcnt < 0) { + /* last writer: disable writer overallocation */ + writer.overallocate = 0; + } + sign = 0; fill = ' '; switch (c) { @@ -778,6 +814,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) pbuf = "%"; len = 1; break; + case 'r': // %r is only for 2/3 code; 3 only code should use %a case 'a': @@ -790,6 +827,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (prec >= 0 && len > prec) len = prec; break; + case 's': // %s is only for 2/3 code; 3 only code should use %b case 'b': @@ -799,6 +837,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (prec >= 0 && len > prec) len = prec; break; + case 'i': case 'd': case 'u': @@ -815,14 +854,24 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (flags & F_ZERO) fill = '0'; break; + case 'e': case 'E': case 'f': case 'F': case 'g': case 'G': - temp = formatfloat(v, flags, prec, c); - if (temp == NULL) + if (width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK))) + { + /* Fast path */ + res = formatfloat(v, flags, prec, c, NULL, &writer, res, 1); + if (res == NULL) + goto error; + continue; + } + + if (!formatfloat(v, flags, prec, c, &temp, NULL, res, 1)) goto error; pbuf = PyBytes_AS_STRING(temp); len = PyBytes_GET_SIZE(temp); @@ -830,12 +879,14 @@ _PyBytes_Format(PyObject *format, PyObject *args) if (flags & F_ZERO) fill = '0'; break; + case 'c': pbuf = &onechar; len = byte_converter(v, &onechar); if (!len) goto error; break; + default: PyErr_Format(PyExc_ValueError, "unsupported format character '%c' (0x%x) " @@ -845,6 +896,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) PyBytes_AsString(format))); goto error; } + if (sign) { if (*pbuf == '-' || *pbuf == '+') { sign = *pbuf++; @@ -859,29 +911,30 @@ _PyBytes_Format(PyObject *format, PyObject *args) } if (width < len) width = len; - if (rescnt - (sign != 0) < width) { - reslen -= rescnt; - rescnt = width + fmtcnt + 100; - reslen += rescnt; - if (reslen < 0) { - Py_DECREF(result); - Py_XDECREF(temp); - return PyErr_NoMemory(); - } - if (_PyBytes_Resize(&result, reslen)) { - Py_XDECREF(temp); - return NULL; - } - res = PyBytes_AS_STRING(result) - + reslen - rescnt; + + alloc = width; + if (sign != 0 && len == width) + alloc++; + if (alloc > 1) { + res = _PyBytesWriter_Prepare(&writer, res, alloc - 1); + if (res == NULL) + goto error; } +#ifdef Py_DEBUG + before = res; +#endif + + /* Write the sign if needed */ if (sign) { if (fill != ' ') *res++ = sign; - rescnt--; if (width > len) width--; } + + /* Write the numeric prefix for "x", "X" and "o" formats + if the alternate form is used. + For example, write "0x" for the "%#x" format. */ if ((flags & F_ALT) && (c == 'x' || c == 'X')) { assert(pbuf[0] == '0'); assert(pbuf[1] == c); @@ -889,18 +942,21 @@ _PyBytes_Format(PyObject *format, PyObject *args) *res++ = *pbuf++; *res++ = *pbuf++; } - rescnt -= 2; width -= 2; if (width < 0) width = 0; len -= 2; } + + /* Pad left with the fill character if needed */ if (width > len && !(flags & F_LJUST)) { - do { - --rescnt; - *res++ = fill; - } while (--width > len); + memset(res, fill, width - len); + res += (width - len); + width = len; } + + /* If padding with spaces: write sign if needed and/or numeric + prefix if the alternate form is used */ if (fill == ' ') { if (sign) *res++ = sign; @@ -912,13 +968,17 @@ _PyBytes_Format(PyObject *format, PyObject *args) *res++ = *pbuf++; } } + + /* Copy bytes */ Py_MEMCPY(res, pbuf, len); res += len; - rescnt -= len; - while (--width >= len) { - --rescnt; - *res++ = ' '; + + /* Pad right with the fill character if needed */ + if (width > len) { + memset(res, ' ', width - len); + res += (width - len); } + if (dict && (argidx < arglen) && c != '%') { PyErr_SetString(PyExc_TypeError, "not all arguments converted during bytes formatting"); @@ -926,22 +986,31 @@ _PyBytes_Format(PyObject *format, PyObject *args) goto error; } Py_XDECREF(temp); + +#ifdef Py_DEBUG + /* check that we computed the exact size for this write */ + assert((res - before) == alloc); +#endif } /* '%' */ + + /* If overallocation was disabled, ensure that it was the last + write. Otherwise, we missed an optimization */ + assert(writer.overallocate || fmtcnt < 0); } /* until end */ + if (argidx < arglen && !dict) { PyErr_SetString(PyExc_TypeError, "not all arguments converted during bytes formatting"); goto error; } + if (args_owned) { Py_DECREF(args); } - if (_PyBytes_Resize(&result, reslen - rescnt)) - return NULL; - return result; + return _PyBytesWriter_Finish(&writer, res); error: - Py_DECREF(result); + _PyBytesWriter_Dealloc(&writer); if (args_owned) { Py_DECREF(args); } -- cgit v0.12 From 53926a1ce2772ea5dc1c4fb8c19f5f9ae461750d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:37:03 +0200 Subject: _PyBytesWriter: rename size attribute to min_size --- Include/bytesobject.h | 5 +++-- Objects/bytesobject.c | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index ffa529b..6cd5a34 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -134,8 +134,9 @@ typedef struct { /* Number of allocated size */ Py_ssize_t allocated; - /* Current size of the buffer (can be smaller than the allocated size) */ - Py_ssize_t size; + /* Minimum number of allocated bytes, + incremented by _PyBytesWriter_Prepare() */ + Py_ssize_t min_size; /* If non-zero, overallocate the buffer (default: 0). */ int overallocate; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ef53288..4a0735f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3821,7 +3821,7 @@ _PyBytesWriter_Init(_PyBytesWriter *writer) { writer->buffer = NULL; writer->allocated = 0; - writer->size = 0; + writer->min_size = 0; writer->overallocate = 0; writer->use_small_buffer = 0; #ifdef Py_DEBUG @@ -3874,7 +3874,7 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) } start = _PyBytesWriter_AsString(writer); - assert(0 <= writer->size && writer->size <= writer->allocated); + assert(0 <= writer->min_size && writer->min_size <= writer->allocated); /* the last byte must always be null */ assert(start[writer->allocated] == 0); @@ -3897,18 +3897,18 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) return str; } - if (writer->size > PY_SSIZE_T_MAX - size) { + if (writer->min_size > PY_SSIZE_T_MAX - size) { PyErr_NoMemory(); _PyBytesWriter_Dealloc(writer); return NULL; } - writer->size += size; + writer->min_size += size; allocated = writer->allocated; - if (writer->size <= allocated) + if (writer->min_size <= allocated) return str; - allocated = writer->size; + allocated = writer->min_size; if (writer->overallocate && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ @@ -3957,7 +3957,7 @@ char* _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) { /* ensure that _PyBytesWriter_Alloc() is only called once */ - assert(writer->size == 0 && writer->buffer == NULL); + assert(writer->min_size == 0 && writer->buffer == NULL); assert(size >= 0); writer->use_small_buffer = 1; -- cgit v0.12 From ad7715891e3c6c51b4860e0d496843ee5417206b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:38:53 +0200 Subject: _PyBytesWriter: simplify code to avoid "prealloc" parameters Substract preallocate bytes from min_size before calling _PyBytesWriter_Prepare(). --- Objects/bytesobject.c | 16 ++++++------- Objects/stringlib/codecs.h | 20 +++++++++------- Objects/unicodeobject.c | 58 ++++++++++++++++++++++------------------------ 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4a0735f..075edf8 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -411,8 +411,7 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) static char* formatfloat(PyObject *v, int flags, int prec, int type, - PyObject **p_result, _PyBytesWriter *writer, char *str, - Py_ssize_t prealloc) + PyObject **p_result, _PyBytesWriter *writer, char *str) { char *p; PyObject *result; @@ -437,11 +436,9 @@ formatfloat(PyObject *v, int flags, int prec, int type, len = strlen(p); if (writer != NULL) { - if ((Py_ssize_t)len > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, len - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, len); + if (str == NULL) + return NULL; Py_MEMCPY(str, p, len); str += len; return str; @@ -865,13 +862,14 @@ _PyBytes_Format(PyObject *format, PyObject *args) && !(flags & (F_SIGN | F_BLANK))) { /* Fast path */ - res = formatfloat(v, flags, prec, c, NULL, &writer, res, 1); + writer.min_size -= 2; /* size preallocated by "%f" */ + res = formatfloat(v, flags, prec, c, NULL, &writer, res); if (res == NULL) goto error; continue; } - if (!formatfloat(v, flags, prec, c, &temp, NULL, res, 1)) + if (!formatfloat(v, flags, prec, c, &temp, NULL, res)) goto error; pbuf = PyBytes_AS_STRING(temp); len = PyBytes_GET_SIZE(temp); diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index ae99d1a..6842f67 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -345,7 +345,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - p = backslashreplace(&writer, max_char_size, p, + /* substract preallocated bytes */ + writer.min_size -= max_char_size * (endpos - startpos); + p = backslashreplace(&writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -353,7 +355,9 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - p = xmlcharrefreplace(&writer, max_char_size, p, + /* substract preallocated bytes */ + writer.min_size -= max_char_size * (endpos - startpos); + p = xmlcharrefreplace(&writer, p, unicode, startpos, endpos); if (p == NULL) goto error; @@ -381,17 +385,17 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, if (!rep) goto error; + /* substract preallocated bytes */ + writer.min_size -= max_char_size; + if (PyBytes_Check(rep)) repsize = PyBytes_GET_SIZE(rep); else repsize = PyUnicode_GET_LENGTH(rep); - if (repsize > max_char_size) { - p = _PyBytesWriter_Prepare(&writer, p, - repsize - max_char_size); - if (p == NULL) - goto error; - } + p = _PyBytesWriter_Prepare(&writer, p, repsize); + if (p == NULL) + goto error; if (PyBytes_Check(rep)) { memcpy(p, PyBytes_AS_STRING(rep), repsize); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a3bbf92..0bcacd8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -565,11 +565,10 @@ unicode_result_unchanged(PyObject *unicode) /* Implementation of the "backslashreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, - char *str, +backslashreplace(_PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { - Py_ssize_t size, i, prealloc; + Py_ssize_t size, i; Py_UCS4 ch; enum PyUnicode_Kind kind; void *data; @@ -600,12 +599,9 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, size += incr; } - prealloc = prealloc_per_char * (collend - collstart); - if (size > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, size - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -636,11 +632,10 @@ backslashreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, /* Implementation of the "xmlcharrefreplace" error handler for 8-bit encodings: ASCII, Latin1, UTF-8, etc. */ static char* -xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, - char *str, +xmlcharrefreplace(_PyBytesWriter *writer, char *str, PyObject *unicode, Py_ssize_t collstart, Py_ssize_t collend) { - Py_ssize_t size, i, prealloc; + Py_ssize_t size, i; Py_UCS4 ch; enum PyUnicode_Kind kind; void *data; @@ -679,12 +674,9 @@ xmlcharrefreplace(_PyBytesWriter *writer, Py_ssize_t prealloc_per_char, size += incr; } - prealloc = prealloc_per_char * (collend - collstart); - if (size > prealloc) { - str = _PyBytesWriter_Prepare(writer, str, size - prealloc); - if (str == NULL) - return NULL; - } + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; /* generate replacement */ for (i = collstart; i < collend; ++i) { @@ -6666,7 +6658,9 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_BACKSLASHREPLACE: - str = backslashreplace(&writer, 1, str, + /* substract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = backslashreplace(&writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -6674,7 +6668,9 @@ unicode_encode_ucs1(PyObject *unicode, break; case _Py_ERROR_XMLCHARREFREPLACE: - str = xmlcharrefreplace(&writer, 1, str, + /* substract preallocated bytes */ + writer.min_size -= (collend - collstart); + str = xmlcharrefreplace(&writer, str, unicode, collstart, collend); if (str == NULL) goto onError; @@ -6705,14 +6701,17 @@ unicode_encode_ucs1(PyObject *unicode, PyUnicode_READY(repunicode) == -1)) goto onError; + /* substract preallocated bytes */ + writer.min_size -= 1; + if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ repsize = PyBytes_Size(repunicode); - if (repsize > 1) { - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; - } + + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; + memcpy(str, PyBytes_AsString(repunicode), repsize); str += repsize; pos = newpos; @@ -6724,11 +6723,10 @@ unicode_encode_ucs1(PyObject *unicode, have+the replacement+the rest of the string, so we won't have to check space for encodable characters) */ repsize = PyUnicode_GET_LENGTH(repunicode); - if (repsize > 1) { - str = _PyBytesWriter_Prepare(&writer, str, repsize-1); - if (str == NULL) - goto onError; - } + + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; /* check if there is anything unencodable in the replacement and copy it to the output */ -- cgit v0.12 From ce179bf6baed91ba84cc3ff647e96287c3b8e2f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 12:57:22 +0200 Subject: Add _PyBytesWriter_WriteBytes() to factorize the code --- Include/bytesobject.h | 7 +++++++ Objects/bytesobject.c | 14 ++++++++++++++ Objects/stringlib/codecs.h | 22 +++++++++++----------- Objects/unicodeobject.c | 8 +++----- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 6cd5a34..2c4c4c4 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -174,6 +174,13 @@ PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size); + +/* Write bytes. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(char*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + char *str, + char *bytes, + Py_ssize_t size); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 075edf8..3aa905c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3995,3 +3995,17 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) return result; } + +char* +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, char *str, + char *bytes, Py_ssize_t size) +{ + str = _PyBytesWriter_Prepare(writer, str, size); + if (str == NULL) + return NULL; + + Py_MEMCPY(str, bytes, size); + str += size; + + return str; +} diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 6842f67..7e8d928 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -388,24 +388,24 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, /* substract preallocated bytes */ writer.min_size -= max_char_size; - if (PyBytes_Check(rep)) - repsize = PyBytes_GET_SIZE(rep); - else - repsize = PyUnicode_GET_LENGTH(rep); - - p = _PyBytesWriter_Prepare(&writer, p, repsize); - if (p == NULL) - goto error; - if (PyBytes_Check(rep)) { - memcpy(p, PyBytes_AS_STRING(rep), repsize); - p += repsize; + p = _PyBytesWriter_WriteBytes(&writer, p, + PyBytes_AS_STRING(rep), + PyBytes_GET_SIZE(rep)); + if (p == NULL) + goto error; } else { /* rep is unicode */ if (PyUnicode_READY(rep) < 0) goto error; + repsize = PyUnicode_GET_LENGTH(rep); + + p = _PyBytesWriter_Prepare(&writer, p, repsize); + if (p == NULL) + goto error; + if (!PyUnicode_IS_ASCII(rep)) { raise_encode_exception(&exc, "utf-8", unicode, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0bcacd8..23b8cc7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6706,14 +6706,12 @@ unicode_encode_ucs1(PyObject *unicode, if (PyBytes_Check(repunicode)) { /* Directly copy bytes result to output. */ - repsize = PyBytes_Size(repunicode); - - str = _PyBytesWriter_Prepare(&writer, str, repsize); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyBytes_AS_STRING(repunicode), + PyBytes_GET_SIZE(repunicode)); if (str == NULL) goto onError; - memcpy(str, PyBytes_AsString(repunicode), repsize); - str += repsize; pos = newpos; Py_DECREF(repunicode); break; -- cgit v0.12 From 6bd525b656f75c9752d39d9c4be1e1b29fa67cdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 13:10:05 +0200 Subject: Optimize error handlers of ASCII and Latin1 encoders when the replacement string is pure ASCII: use _PyBytesWriter_WriteBytes(), don't check individual character. Cleanup unicode_encode_ucs1(): * Rename repunicode to rep * Clear rep object on error * Factorize code between bytes and unicode path --- Objects/stringlib/codecs.h | 18 +++++------- Objects/unicodeobject.c | 72 +++++++++++++++++++++++++--------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/Objects/stringlib/codecs.h b/Objects/stringlib/codecs.h index 7e8d928..2beb604 100644 --- a/Objects/stringlib/codecs.h +++ b/Objects/stringlib/codecs.h @@ -311,7 +311,7 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, #if STRINGLIB_SIZEOF_CHAR > 1 else if (Py_UNICODE_IS_SURROGATE(ch)) { Py_ssize_t startpos, endpos, newpos; - Py_ssize_t repsize, k; + Py_ssize_t k; if (error_handler == _Py_ERROR_UNKNOWN) error_handler = get_error_handler(errors); @@ -392,20 +392,12 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, p = _PyBytesWriter_WriteBytes(&writer, p, PyBytes_AS_STRING(rep), PyBytes_GET_SIZE(rep)); - if (p == NULL) - goto error; } else { /* rep is unicode */ if (PyUnicode_READY(rep) < 0) goto error; - repsize = PyUnicode_GET_LENGTH(rep); - - p = _PyBytesWriter_Prepare(&writer, p, repsize); - if (p == NULL) - goto error; - if (!PyUnicode_IS_ASCII(rep)) { raise_encode_exception(&exc, "utf-8", unicode, @@ -415,9 +407,13 @@ STRINGLIB(utf8_encoder)(PyObject *unicode, } assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); - memcpy(p, PyUnicode_DATA(rep), repsize); - p += repsize; + p = _PyBytesWriter_WriteBytes(&writer, p, + PyUnicode_DATA(rep), + PyUnicode_GET_LENGTH(rep)); } + + if (p == NULL) + goto error; Py_CLEAR(rep); i = newpos; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 23b8cc7..35df747 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6599,6 +6599,7 @@ unicode_encode_ucs1(PyObject *unicode, PyObject *error_handler_obj = NULL; PyObject *exc = NULL; _Py_error_handler error_handler = _Py_ERROR_UNKNOWN; + PyObject *rep = NULL; /* output object */ _PyBytesWriter writer; @@ -6627,8 +6628,7 @@ unicode_encode_ucs1(PyObject *unicode, ++pos; } else { - PyObject *repunicode; - Py_ssize_t repsize, newpos, i; + Py_ssize_t newpos, i; /* startpos for collecting unencodable chars */ Py_ssize_t collstart = pos; Py_ssize_t collend = collstart + 1; @@ -6694,52 +6694,59 @@ unicode_encode_ucs1(PyObject *unicode, /* fallback to general error handling */ default: - repunicode = unicode_encode_call_errorhandler(errors, &error_handler_obj, - encoding, reason, unicode, &exc, - collstart, collend, &newpos); - if (repunicode == NULL || (PyUnicode_Check(repunicode) && - PyUnicode_READY(repunicode) == -1)) + rep = unicode_encode_call_errorhandler(errors, &error_handler_obj, + encoding, reason, unicode, &exc, + collstart, collend, &newpos); + if (rep == NULL) goto onError; /* substract preallocated bytes */ writer.min_size -= 1; - if (PyBytes_Check(repunicode)) { + if (PyBytes_Check(rep)) { /* Directly copy bytes result to output. */ str = _PyBytesWriter_WriteBytes(&writer, str, - PyBytes_AS_STRING(repunicode), - PyBytes_GET_SIZE(repunicode)); + PyBytes_AS_STRING(rep), + PyBytes_GET_SIZE(rep)); if (str == NULL) goto onError; - - pos = newpos; - Py_DECREF(repunicode); - break; } + else { + assert(PyUnicode_Check(rep)); - /* need more space? (at least enough for what we - have+the replacement+the rest of the string, so - we won't have to check space for encodable characters) */ - repsize = PyUnicode_GET_LENGTH(repunicode); + if (PyUnicode_READY(rep) < 0) + goto onError; - str = _PyBytesWriter_Prepare(&writer, str, repsize); - if (str == NULL) - goto onError; + if (PyUnicode_IS_ASCII(rep)) { + /* Fast path: all characters are smaller than limit */ + assert(limit >= 128); + assert(PyUnicode_KIND(rep) == PyUnicode_1BYTE_KIND); + str = _PyBytesWriter_WriteBytes(&writer, str, + PyUnicode_DATA(rep), + PyUnicode_GET_LENGTH(rep)); + } + else { + Py_ssize_t repsize = PyUnicode_GET_LENGTH(rep); - /* check if there is anything unencodable in the replacement - and copy it to the output */ - for (i = 0; repsize-->0; ++i, ++str) { - ch = PyUnicode_READ_CHAR(repunicode, i); - if (ch >= limit) { - raise_encode_exception(&exc, encoding, unicode, - pos, pos+1, reason); - Py_DECREF(repunicode); - goto onError; + str = _PyBytesWriter_Prepare(&writer, str, repsize); + if (str == NULL) + goto onError; + + /* check if there is anything unencodable in the + replacement and copy it to the output */ + for (i = 0; repsize-->0; ++i, ++str) { + ch = PyUnicode_READ_CHAR(rep, i); + if (ch >= limit) { + raise_encode_exception(&exc, encoding, unicode, + pos, pos+1, reason); + goto onError; + } + *str = (char)ch; + } } - *str = (char)ch; } pos = newpos; - Py_DECREF(repunicode); + Py_CLEAR(rep); } /* If overallocation was disabled, ensure that it was the last @@ -6753,6 +6760,7 @@ unicode_encode_ucs1(PyObject *unicode, return _PyBytesWriter_Finish(&writer, str); onError: + Py_XDECREF(rep); _PyBytesWriter_Dealloc(&writer); Py_XDECREF(error_handler_obj); Py_XDECREF(exc); -- cgit v0.12 From be75b8cf2310e0be0f32590a1c0df919058002e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 22:43:24 +0200 Subject: Issue #25349: Optimize bytes % int Optimize bytes.__mod__(args) for integere formats: %d (%i, %u), %o, %x and %X. _PyBytesWriter is now used to format directly the integer into the writer buffer, instead of using a temporary bytes object. Formatting is between 30% and 50% faster on a microbenchmark. --- Include/longobject.h | 7 +++ Objects/bytesobject.c | 36 +++++++++++++++ Objects/longobject.c | 120 ++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 136 insertions(+), 27 deletions(-) diff --git a/Include/longobject.h b/Include/longobject.h index aed59ce..ab92495 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -182,6 +182,13 @@ PyAPI_FUNC(int) _PyLong_FormatWriter( int base, int alternate); +PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( + _PyBytesWriter *writer, + char *str, + PyObject *obj, + int base, + int alternate); + /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 3aa905c..fd46048 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -841,6 +841,42 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'o': case 'x': case 'X': + if (PyLong_CheckExact(v) + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK)) + && c != 'X') + { + /* Fast path */ + int alternate = flags & F_ALT; + int base; + + switch(c) + { + default: + assert(0 && "'type' not in [diuoxX]"); + case 'd': + case 'i': + case 'u': + base = 10; + break; + case 'o': + base = 8; + break; + case 'x': + case 'X': + base = 16; + break; + } + + /* Fast path */ + writer.min_size -= 2; /* size preallocated by "%d" */ + res = _PyLong_FormatBytesWriter(&writer, res, + v, base, alternate); + if (res == NULL) + goto error; + continue; + } + temp = formatlong(v, flags, prec, c); if (!temp) goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 5f22455..00f5d95 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1582,7 +1582,9 @@ divrem1(PyLongObject *a, digit n, digit *prem) static int long_to_decimal_string_internal(PyObject *aa, PyObject **p_output, - _PyUnicodeWriter *writer) + _PyUnicodeWriter *writer, + _PyBytesWriter *bytes_writer, + char **bytes_str) { PyLongObject *scratch, *a; PyObject *str; @@ -1664,6 +1666,13 @@ long_to_decimal_string_internal(PyObject *aa, kind = writer->kind; str = NULL; } + else if (bytes_writer) { + *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); + if (*bytes_str == NULL) { + Py_DECREF(scratch); + return -1; + } + } else { str = PyUnicode_New(strlen, '9'); if (str == NULL) { @@ -1673,13 +1682,8 @@ long_to_decimal_string_internal(PyObject *aa, kind = PyUnicode_KIND(str); } -#define WRITE_DIGITS(TYPE) \ +#define WRITE_DIGITS(p) \ do { \ - if (writer) \ - p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ - else \ - p = (TYPE*)PyUnicode_DATA(str) + strlen; \ - \ /* pout[0] through pout[size-2] contribute exactly \ _PyLong_DECIMAL_SHIFT digits each */ \ for (i=0; i < size - 1; i++) { \ @@ -1699,6 +1703,16 @@ long_to_decimal_string_internal(PyObject *aa, /* and sign */ \ if (negative) \ *--p = '-'; \ + } while (0) + +#define WRITE_UNICODE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + strlen; \ + else \ + p = (TYPE*)PyUnicode_DATA(str) + strlen; \ + \ + WRITE_DIGITS(p); \ \ /* check we've counted correctly */ \ if (writer) \ @@ -1708,25 +1722,34 @@ long_to_decimal_string_internal(PyObject *aa, } while (0) /* fill the string right-to-left */ - if (kind == PyUnicode_1BYTE_KIND) { + if (bytes_writer) { + char *p = *bytes_str + strlen; + WRITE_DIGITS(p); + assert(p == *bytes_str); + } + else if (kind == PyUnicode_1BYTE_KIND) { Py_UCS1 *p; - WRITE_DIGITS(Py_UCS1); + WRITE_UNICODE_DIGITS(Py_UCS1); } else if (kind == PyUnicode_2BYTE_KIND) { Py_UCS2 *p; - WRITE_DIGITS(Py_UCS2); + WRITE_UNICODE_DIGITS(Py_UCS2); } else { Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - WRITE_DIGITS(Py_UCS4); + WRITE_UNICODE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS +#undef WRITE_UNICODE_DIGITS Py_DECREF(scratch); if (writer) { writer->pos += strlen; } + else if (bytes_writer) { + (*bytes_str) += strlen; + } else { assert(_PyUnicode_CheckConsistency(str, 1)); *p_output = (PyObject *)str; @@ -1738,7 +1761,7 @@ static PyObject * long_to_decimal_string(PyObject *aa) { PyObject *v; - if (long_to_decimal_string_internal(aa, &v, NULL) == -1) + if (long_to_decimal_string_internal(aa, &v, NULL, NULL, NULL) == -1) return NULL; return v; } @@ -1750,7 +1773,8 @@ long_to_decimal_string(PyObject *aa) static int long_format_binary(PyObject *aa, int base, int alternate, - PyObject **p_output, _PyUnicodeWriter *writer) + PyObject **p_output, _PyUnicodeWriter *writer, + _PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *a = (PyLongObject *)aa; PyObject *v; @@ -1812,6 +1836,11 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = writer->kind; v = NULL; } + else if (writer) { + *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); + if (*bytes_str == NULL) + return -1; + } else { v = PyUnicode_New(sz, 'x'); if (v == NULL) @@ -1819,13 +1848,8 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = PyUnicode_KIND(v); } -#define WRITE_DIGITS(TYPE) \ +#define WRITE_DIGITS(p) \ do { \ - if (writer) \ - p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ - else \ - p = (TYPE*)PyUnicode_DATA(v) + sz; \ - \ if (size_a == 0) { \ *--p = '0'; \ } \ @@ -1860,30 +1884,50 @@ long_format_binary(PyObject *aa, int base, int alternate, } \ if (negative) \ *--p = '-'; \ + } while (0) + +#define WRITE_UNICODE_DIGITS(TYPE) \ + do { \ + if (writer) \ + p = (TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos + sz; \ + else \ + p = (TYPE*)PyUnicode_DATA(v) + sz; \ + \ + WRITE_DIGITS(p); \ + \ if (writer) \ assert(p == ((TYPE*)PyUnicode_DATA(writer->buffer) + writer->pos)); \ else \ assert(p == (TYPE*)PyUnicode_DATA(v)); \ } while (0) - if (kind == PyUnicode_1BYTE_KIND) { + if (bytes_writer) { + char *p = *bytes_str + sz; + WRITE_DIGITS(p); + assert(p == *bytes_str); + } + else if (kind == PyUnicode_1BYTE_KIND) { Py_UCS1 *p; - WRITE_DIGITS(Py_UCS1); + WRITE_UNICODE_DIGITS(Py_UCS1); } else if (kind == PyUnicode_2BYTE_KIND) { Py_UCS2 *p; - WRITE_DIGITS(Py_UCS2); + WRITE_UNICODE_DIGITS(Py_UCS2); } else { Py_UCS4 *p; assert (kind == PyUnicode_4BYTE_KIND); - WRITE_DIGITS(Py_UCS4); + WRITE_UNICODE_DIGITS(Py_UCS4); } #undef WRITE_DIGITS +#undef WRITE_UNICODE_DIGITS if (writer) { writer->pos += sz; } + else if (bytes_writer) { + (*bytes_str) += sz; + } else { assert(_PyUnicode_CheckConsistency(v, 1)); *p_output = v; @@ -1897,9 +1941,9 @@ _PyLong_Format(PyObject *obj, int base) PyObject *str; int err; if (base == 10) - err = long_to_decimal_string_internal(obj, &str, NULL); + err = long_to_decimal_string_internal(obj, &str, NULL, NULL, NULL); else - err = long_format_binary(obj, base, 1, &str, NULL); + err = long_format_binary(obj, base, 1, &str, NULL, NULL, NULL); if (err == -1) return NULL; return str; @@ -1911,9 +1955,31 @@ _PyLong_FormatWriter(_PyUnicodeWriter *writer, int base, int alternate) { if (base == 10) - return long_to_decimal_string_internal(obj, NULL, writer); + return long_to_decimal_string_internal(obj, NULL, writer, + NULL, NULL); else - return long_format_binary(obj, base, alternate, NULL, writer); + return long_format_binary(obj, base, alternate, NULL, writer, + NULL, NULL); +} + +char* +_PyLong_FormatBytesWriter(_PyBytesWriter *writer, char *str, + PyObject *obj, + int base, int alternate) +{ + char *str2; + int res; + str2 = str; + if (base == 10) + res = long_to_decimal_string_internal(obj, NULL, NULL, + writer, &str2); + else + res = long_format_binary(obj, base, alternate, NULL, NULL, + writer, &str2); + if (res < 0) + return NULL; + assert(str2 != NULL); + return str2; } /* Table of digit values for 8-bit string -> integer conversion. -- cgit v0.12 From 0cdad1e2bc8794cdbba8a89e2a4177a7dedec313 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 9 Oct 2015 22:50:36 +0200 Subject: Issue #25349: Add fast path for b'%c' % int Optimize also %% formater. --- Lib/test/test_format.py | 2 ++ Objects/bytesobject.c | 25 +++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 9b13632..9924fde 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -300,6 +300,8 @@ class FormatTest(unittest.TestCase): testcommon(b"%c", 7, b"\x07") testcommon(b"%c", b"Z", b"Z") testcommon(b"%c", bytearray(b"Z"), b"Z") + testcommon(b"%5c", 65, b" A") + testcommon(b"%-5c", 65, b"A ") # %b will insert a series of bytes, either from a type that supports # the Py_buffer protocol, or something that has a __bytes__ method class FakeBytes(object): diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index fd46048..a75c54d 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -808,9 +808,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) fill = ' '; switch (c) { case '%': - pbuf = "%"; - len = 1; - break; + *res++ = '%'; + continue; case 'r': // %r is only for 2/3 code; 3 only code should use %a @@ -842,9 +841,9 @@ _PyBytes_Format(PyObject *format, PyObject *args) case 'x': case 'X': if (PyLong_CheckExact(v) - && width == -1 && prec == -1 - && !(flags & (F_SIGN | F_BLANK)) - && c != 'X') + && width == -1 && prec == -1 + && !(flags & (F_SIGN | F_BLANK)) + && c != 'X') { /* Fast path */ int alternate = flags & F_ALT; @@ -869,7 +868,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) } /* Fast path */ - writer.min_size -= 2; /* size preallocated by "%d" */ + writer.min_size -= 2; /* size preallocated for "%d" */ res = _PyLong_FormatBytesWriter(&writer, res, v, base, alternate); if (res == NULL) @@ -898,7 +897,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) && !(flags & (F_SIGN | F_BLANK))) { /* Fast path */ - writer.min_size -= 2; /* size preallocated by "%f" */ + writer.min_size -= 2; /* size preallocated for "%f" */ res = formatfloat(v, flags, prec, c, NULL, &writer, res); if (res == NULL) goto error; @@ -919,6 +918,11 @@ _PyBytes_Format(PyObject *format, PyObject *args) len = byte_converter(v, &onechar); if (!len) goto error; + if (width == -1) { + /* Fast path */ + *res++ = onechar; + continue; + } break; default: @@ -949,8 +953,9 @@ _PyBytes_Format(PyObject *format, PyObject *args) alloc = width; if (sign != 0 && len == width) alloc++; - if (alloc > 1) { - res = _PyBytesWriter_Prepare(&writer, res, alloc - 1); + /* 2: size preallocated for %s */ + if (alloc > 2) { + res = _PyBytesWriter_Prepare(&writer, res, alloc - 2); if (res == NULL) goto error; } -- cgit v0.12 From 0d554d7ef159761439ade414fcd028262eae656c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 10 Oct 2015 22:42:18 +0300 Subject: Issue #24164: Objects that need calling ``__new__`` with keyword arguments, can now be pickled using pickle protocols older than protocol version 4. --- Doc/whatsnew/3.6.rst | 9 ++++++ Lib/pickle.py | 17 +++++++---- Lib/test/pickletester.py | 6 ++-- Misc/NEWS | 3 ++ Modules/_pickle.c | 78 +++++++++++++++++++++++++++++++++++++++++------- Objects/typeobject.c | 16 ++-------- 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 50d2a6a..b735e9c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -112,6 +112,15 @@ indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` (Contributed by Joe Jevnik in :issue:`24379`.) +pickle +------ + +Objects that need calling ``__new__`` with keyword arguments, can now be pickled +using :ref:`pickle protocols ` older than protocol version 4. +Protocol version 4 already supports this case. (Contributed by Serhiy +Storchaka in :issue:`24164`.) + + rlcomplete ---------- diff --git a/Lib/pickle.py b/Lib/pickle.py index e93057a..d41753d 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -27,6 +27,7 @@ from types import FunctionType from copyreg import dispatch_table from copyreg import _extension_registry, _inverted_registry, _extension_cache from itertools import islice +from functools import partial import sys from sys import maxsize from struct import pack, unpack @@ -544,7 +545,7 @@ class _Pickler: write = self.write func_name = getattr(func, "__name__", "") - if self.proto >= 4 and func_name == "__newobj_ex__": + if self.proto >= 2 and func_name == "__newobj_ex__": cls, args, kwargs = args if not hasattr(cls, "__new__"): raise PicklingError("args[0] from {} args has no __new__" @@ -552,10 +553,16 @@ class _Pickler: if obj is not None and cls is not obj.__class__: raise PicklingError("args[0] from {} args has the wrong class" .format(func_name)) - save(cls) - save(args) - save(kwargs) - write(NEWOBJ_EX) + if self.proto >= 4: + save(cls) + save(args) + save(kwargs) + write(NEWOBJ_EX) + else: + func = partial(cls.__new__, cls, *args, **kwargs) + save(func) + save(()) + write(REDUCE) elif self.proto >= 2 and func_name == "__newobj__": # A __reduce__ implementation can direct protocol 2 or newer to # use the more efficient NEWOBJ opcode, while still diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 2ef48e6..fd641e3 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1580,16 +1580,14 @@ class AbstractPickleTests(unittest.TestCase): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): - if 2 <= proto < 4: - self.assertRaises(ValueError, self.dumps, x, proto) - continue s = self.dumps(x, proto) if proto < 1: self.assertIn(b'\nL64206', s) # LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 + elif proto < 4: + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE else: - assert proto >= 4 self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE self.assertFalse(opcode_in_pickle(pickle.NEWOBJ, s)) self.assertEqual(opcode_in_pickle(pickle.NEWOBJ_EX, s), diff --git a/Misc/NEWS b/Misc/NEWS index cb02aae..802f4b6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- Issue #24164: Objects that need calling ``__new__`` with keyword arguments, + can now be pickled using pickle protocols older than protocol version 4. + - Issue #25364: zipfile now works in threads disabled builds. - Issue #25328: smtpd's SMTPChannel now correctly raises a ValueError if both diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f3b73f1..abaf4e5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -153,6 +153,9 @@ typedef struct { PyObject *codecs_encode; /* builtins.getattr, used for saving nested names with protocol < 4 */ PyObject *getattr; + /* functools.partial, used for implementing __newobj_ex__ with protocols + 2 and 3 */ + PyObject *partial; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -200,6 +203,7 @@ _Pickle_InitState(PickleState *st) PyObject *copyreg = NULL; PyObject *compat_pickle = NULL; PyObject *codecs = NULL; + PyObject *functools = NULL; builtins = PyEval_GetBuiltins(); if (builtins == NULL) @@ -314,12 +318,21 @@ _Pickle_InitState(PickleState *st) } Py_CLEAR(codecs); + functools = PyImport_ImportModule("functools"); + if (!functools) + goto error; + st->partial = PyObject_GetAttrString(functools, "partial"); + if (!st->partial) + goto error; + Py_CLEAR(functools); + return 0; error: Py_CLEAR(copyreg); Py_CLEAR(compat_pickle); Py_CLEAR(codecs); + Py_CLEAR(functools); _Pickle_ClearState(st); return -1; } @@ -3533,11 +3546,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) PyErr_Clear(); } else if (PyUnicode_Check(name)) { - if (self->proto >= 4) { - _Py_IDENTIFIER(__newobj_ex__); - use_newobj_ex = PyUnicode_Compare( - name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0; - } + _Py_IDENTIFIER(__newobj_ex__); + use_newobj_ex = PyUnicode_Compare( + name, _PyUnicode_FromId(&PyId___newobj_ex__)) == 0; if (!use_newobj_ex) { _Py_IDENTIFIER(__newobj__); use_newobj = PyUnicode_Compare( @@ -3581,11 +3592,58 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } - if (save(self, cls, 0) < 0 || - save(self, args, 0) < 0 || - save(self, kwargs, 0) < 0 || - _Pickler_Write(self, &newobj_ex_op, 1) < 0) { - return -1; + if (self->proto >= 4) { + if (save(self, cls, 0) < 0 || + save(self, args, 0) < 0 || + save(self, kwargs, 0) < 0 || + _Pickler_Write(self, &newobj_ex_op, 1) < 0) { + return -1; + } + } + else { + PyObject *newargs; + PyObject *cls_new; + Py_ssize_t i; + _Py_IDENTIFIER(__new__); + + newargs = PyTuple_New(Py_SIZE(args) + 2); + if (newargs == NULL) + return -1; + + cls_new = _PyObject_GetAttrId(cls, &PyId___new__); + if (cls_new == NULL) { + Py_DECREF(newargs); + return -1; + } + PyTuple_SET_ITEM(newargs, 0, cls_new); + Py_INCREF(cls); + PyTuple_SET_ITEM(newargs, 1, cls); + for (i = 0; i < Py_SIZE(args); i++) { + PyObject *item = PyTuple_GET_ITEM(args, i); + Py_INCREF(item); + PyTuple_SET_ITEM(newargs, i + 2, item); + } + + callable = PyObject_Call(st->partial, newargs, kwargs); + Py_DECREF(newargs); + if (callable == NULL) + return -1; + + newargs = PyTuple_New(0); + if (newargs == NULL) { + Py_DECREF(callable); + return -1; + } + + if (save(self, callable, 0) < 0 || + save(self, newargs, 0) < 0 || + _Pickler_Write(self, &reduce_op, 1) < 0) { + Py_DECREF(newargs); + Py_DECREF(callable); + return -1; + } + Py_DECREF(newargs); + Py_DECREF(callable); } } else if (use_newobj) { diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4b091f5..bf0d30c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4101,7 +4101,7 @@ _PyObject_GetItemsIter(PyObject *obj, PyObject **listitems, } static PyObject * -reduce_newobj(PyObject *obj, int proto) +reduce_newobj(PyObject *obj) { PyObject *args = NULL, *kwargs = NULL; PyObject *copyreg; @@ -4153,7 +4153,7 @@ reduce_newobj(PyObject *obj, int proto) } Py_DECREF(args); } - else if (proto >= 4) { + else { _Py_IDENTIFIER(__newobj_ex__); newobj = _PyObject_GetAttrId(copyreg, &PyId___newobj_ex__); @@ -4171,16 +4171,6 @@ reduce_newobj(PyObject *obj, int proto) return NULL; } } - else { - PyErr_SetString(PyExc_ValueError, - "must use protocol 4 or greater to copy this " - "object; since __getnewargs_ex__ returned " - "keyword arguments."); - Py_DECREF(args); - Py_DECREF(kwargs); - Py_DECREF(copyreg); - return NULL; - } state = _PyObject_GetState(obj); if (state == NULL) { @@ -4225,7 +4215,7 @@ _common_reduce(PyObject *self, int proto) PyObject *copyreg, *res; if (proto >= 2) - return reduce_newobj(self, proto); + return reduce_newobj(self); copyreg = import_copyreg(); if (!copyreg) -- cgit v0.12 From a7f63009d6df9c6ae57e6835857d6c5fda024930 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 10 Oct 2015 23:56:02 -0400 Subject: Minor tweak. Make the maxlen comparisons a little more clear and consistent. --- Modules/_collectionsmodule.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4ea9140..52f40b2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -281,7 +281,7 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); static void deque_trim_right(dequeobject *deque) { - if (deque->maxlen != -1 && Py_SIZE(deque) > deque->maxlen) { + if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { PyObject *rv = deque_pop(deque, NULL); assert(rv != NULL); assert(Py_SIZE(deque) <= deque->maxlen); @@ -292,7 +292,7 @@ deque_trim_right(dequeobject *deque) static void deque_trim_left(dequeobject *deque) { - if (deque->maxlen != -1 && Py_SIZE(deque) > deque->maxlen) { + if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { PyObject *rv = deque_popleft(deque, NULL); assert(rv != NULL); assert(Py_SIZE(deque) <= deque->maxlen); @@ -385,7 +385,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen != -1); + int trim = (deque->maxlen >= 0); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -447,7 +447,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen != -1); + int trim = (deque->maxlen >= 0); /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -686,7 +686,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) /* common case, repeating a single element */ PyObject *item = deque->leftblock->data[deque->leftindex]; - if (deque->maxlen != -1 && n > deque->maxlen) + if (deque->maxlen >= 0 && n > deque->maxlen) n = deque->maxlen; if (n > MAX_DEQUE_LEN) @@ -1355,7 +1355,7 @@ deque_repr(PyObject *deque) Py_ReprLeave(deque); return NULL; } - if (((dequeobject *)deque)->maxlen != -1) + if (((dequeobject *)deque)->maxlen >= 0) result = PyUnicode_FromFormat("deque(%R, maxlen=%zd)", aslist, ((dequeobject *)deque)->maxlen); else -- cgit v0.12 From 647dac9d6fc08b9fad14c10dabe7cc8ee48c8553 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 09:47:17 +0200 Subject: Close #25368: Fix test_eintr when Python is compiled without thread support --- Lib/test/eintrdata/eintr_tester.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 443ccd5..9de2b6b 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -52,7 +52,8 @@ class EINTRBaseTest(unittest.TestCase): cls.signal_period) # Issue #25277: Use faulthandler to try to debug a hang on FreeBSD - faulthandler.dump_traceback_later(10 * 60, exit=True) + if hasattr(faulthandler, 'dump_traceback_later'): + faulthandler.dump_traceback_later(10 * 60, exit=True) @classmethod def stop_alarm(cls): @@ -62,7 +63,8 @@ class EINTRBaseTest(unittest.TestCase): def tearDownClass(cls): cls.stop_alarm() signal.signal(signal.SIGALRM, cls.orig_handler) - faulthandler.cancel_dump_traceback_later() + if hasattr(faulthandler, 'cancel_dump_traceback_later'): + faulthandler.cancel_dump_traceback_later() @classmethod def _sleep(cls): -- cgit v0.12 From 92f0113701c6404f9e1ec90e3260084fc6a2ef09 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 09:54:42 +0200 Subject: Close #24784: Fix compilation without thread support Add "#ifdef WITH_THREAD" around cals to: * PyGILState_Check() * _PyImport_AcquireLock() * _PyImport_ReleaseLock() --- Modules/_posixsubprocess.c | 12 ++++++++++-- Modules/socketmodule.c | 2 ++ Python/fileutils.c | 6 ++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index a327fc5..800b301 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -549,7 +549,9 @@ subprocess_fork_exec(PyObject* self, PyObject *args) int need_to_reenable_gc = 0; char *const *exec_array, *const *argv = NULL, *const *envp = NULL; Py_ssize_t arg_num; +#ifdef WITH_THREAD int import_lock_held = 0; +#endif if (!PyArg_ParseTuple( args, "OOpOOOiiiiiiiiiiO:fork_exec", @@ -644,8 +646,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) preexec_fn_args_tuple = PyTuple_New(0); if (!preexec_fn_args_tuple) goto cleanup; +#ifdef WITH_THREAD _PyImport_AcquireLock(); import_lock_held = 1; +#endif } if (cwd_obj != Py_None) { @@ -688,12 +692,14 @@ subprocess_fork_exec(PyObject* self, PyObject *args) /* Capture the errno exception before errno can be clobbered. */ PyErr_SetFromErrno(PyExc_OSError); } - if (preexec_fn != Py_None && - _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { +#ifdef WITH_THREAD + if (preexec_fn != Py_None + && _PyImport_ReleaseLock() < 0 && !PyErr_Occurred()) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); } import_lock_held = 0; +#endif /* Parent process */ if (envp) @@ -716,8 +722,10 @@ subprocess_fork_exec(PyObject* self, PyObject *args) return PyLong_FromPid(pid); cleanup: +#ifdef WITH_THREAD if (import_lock_held) _PyImport_ReleaseLock(); +#endif if (envp) _Py_FreeCharPArray(envp); if (argv) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index d9c70f8..bae9634 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -719,8 +719,10 @@ sock_call_ex(PySocketSockObject *s, int deadline_initialized = 0; int res; +#ifdef WITH_THREAD /* sock_call() must be called with the GIL held. */ assert(PyGILState_Check()); +#endif /* outer loop to retry select() when select() is interrupted by a signal or to retry select()+sock_func() on false positive (see above) */ diff --git a/Python/fileutils.c b/Python/fileutils.c index bccd321..079918c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -986,8 +986,10 @@ _Py_open_impl(const char *pathname, int flags, int gil_held) int _Py_open(const char *pathname, int flags) { +#ifdef WITH_THREAD /* _Py_open() must be called with the GIL held. */ assert(PyGILState_Check()); +#endif return _Py_open_impl(pathname, flags, 1); } @@ -1080,7 +1082,9 @@ _Py_fopen_obj(PyObject *path, const char *mode) wchar_t wmode[10]; int usize; +#ifdef WITH_THREAD assert(PyGILState_Check()); +#endif if (!PyUnicode_Check(path)) { PyErr_Format(PyExc_TypeError, @@ -1108,7 +1112,9 @@ _Py_fopen_obj(PyObject *path, const char *mode) PyObject *bytes; char *path_bytes; +#ifdef WITH_THREAD assert(PyGILState_Check()); +#endif if (!PyUnicode_FSConverter(path, &bytes)) return NULL; -- cgit v0.12 From 4967146c8d6f68480c5ca4ed73940607dd1d9286 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:03:28 +0200 Subject: Close #25369: Fix test_regrtest without thread support --- Lib/test/test_regrtest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 2e33555..0f00698 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -7,6 +7,7 @@ Note: test_regrtest cannot be run twice in parallel. import argparse import faulthandler import getopt +import io import os.path import platform import re @@ -433,7 +434,9 @@ class ProgramsTestCase(BaseTestCase): self.tests = [self.create_test() for index in range(self.NTEST)] self.python_args = ['-Wd', '-E', '-bb'] - self.regrtest_args = ['-uall', '-rwW', '--timeout', '3600', '-j4'] + self.regrtest_args = ['-uall', '-rwW'] + if hasattr(faulthandler, 'dump_traceback_later'): + self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': self.regrtest_args.append('-n') -- cgit v0.12 From 14b4662e18189132e343781070290bf5729452b5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:04:26 +0200 Subject: test_regrtest: catch stderr in test_nowindows() Check also that the deprecation warning is emited. --- Lib/test/test_regrtest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 0f00698..16a29b1 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -5,6 +5,7 @@ Note: test_regrtest cannot be run twice in parallel. """ import argparse +import contextlib import faulthandler import getopt import io @@ -247,8 +248,11 @@ class ParseArgsTestCase(unittest.TestCase): def test_nowindows(self): for opt in '-n', '--nowindows': with self.subTest(opt=opt): - ns = libregrtest._parse_args([opt]) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + ns = libregrtest._parse_args([opt]) self.assertTrue(ns.nowindows) + err = stderr.getvalue() + self.assertIn('the --nowindows (-n) option is deprecated', err) def test_forever(self): for opt in '-F', '--forever': -- cgit v0.12 From 3909e58994f929abacc20e1f32b63e30109f942b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 10:37:25 +0200 Subject: Close #25373: Fix regrtest --slow with interrupted test * Fix accumulate_result(): don't use time on interrupted and failed test * Add unit test for interrupted test * Add unit test on --slow with interrupted test, with and without multiprocessing --- Lib/test/libregrtest/main.py | 12 ++++++++---- Lib/test/test_regrtest.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index aa95b21..82788ad 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -7,10 +7,11 @@ import sys import sysconfig import tempfile import textwrap +from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, - STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED) -from test.libregrtest.cmdline import _parse_args + STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, + INTERRUPTED, CHILD_ERROR) from test.libregrtest.setup import setup_tests from test import support try: @@ -87,7 +88,8 @@ class Regrtest: def accumulate_result(self, test, result): ok, test_time = result - self.test_times.append((test_time, test)) + if ok not in (CHILD_ERROR, INTERRUPTED): + self.test_times.append((test_time, test)) if ok == PASSED: self.good.append(test) elif ok == FAILED: @@ -291,10 +293,12 @@ class Regrtest: else: try: result = runtest(self.ns, test) - self.accumulate_result(test, result) except KeyboardInterrupt: + self.accumulate_result(test, (INTERRUPTED, None)) self.interrupted = True break + else: + self.accumulate_result(test, result) if self.ns.findleaks: gc.collect() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 16a29b1..ab7741f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -24,6 +24,16 @@ Py_DEBUG = hasattr(sys, 'getobjects') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) +TEST_INTERRUPTED = textwrap.dedent(""" + from signal import SIGINT + try: + from _testcapi import raise_signal + raise_signal(SIGINT) + except ImportError: + import os + os.kill(os.getpid(), SIGINT) + """) + class ParseArgsTestCase(unittest.TestCase): """ @@ -340,16 +350,19 @@ class BaseTestCase(unittest.TestCase): return list(match.group(1) for match in parser) def check_executed_tests(self, output, tests, skipped=(), failed=(), - randomize=False): + omitted=(), randomize=False): if isinstance(tests, str): tests = [tests] if isinstance(skipped, str): skipped = [skipped] if isinstance(failed, str): failed = [failed] + if isinstance(omitted, str): + omitted = [omitted] ntest = len(tests) nskipped = len(skipped) nfailed = len(failed) + nomitted = len(omitted) executed = self.parse_executed_tests(output) if randomize: @@ -375,7 +388,11 @@ class BaseTestCase(unittest.TestCase): regex = list_regex('%s test%s failed', failed) self.check_line(output, regex) - good = ntest - nskipped - nfailed + if omitted: + regex = list_regex('%s test%s omitted', omitted) + self.check_line(output, regex) + + good = ntest - nskipped - nfailed - nomitted if good: regex = r'%s test%s OK\.$' % (good, plural(good)) if not skipped and not failed and good > 1: @@ -607,6 +624,12 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + def test_interrupted(self): + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + output = self.run_tests(test, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + def test_slow(self): # test --slow tests = [self.create_test() for index in range(3)] @@ -617,6 +640,22 @@ class ArgsTestCase(BaseTestCase): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) + def test_slow_interrupted(self): + # Issue #25373: test --slow with an interrupted test + code = TEST_INTERRUPTED + test = self.create_test("sigint", code=code) + + for multiprocessing in (False, True): + if multiprocessing: + args = ("--slow", "-j2", test) + else: + args = ("--slow", test) + output = self.run_tests(*args, exitcode=1) + self.check_executed_tests(output, test, omitted=test) + regex = ('10 slowest tests:\n') + self.check_line(output, regex) + self.check_line(output, 'Test suite interrupted by signal SIGINT.') + def test_coverage(self): # test --coverage test = self.create_test() -- cgit v0.12 From e84c97656899750a8e6b01bef2cfc01b31d60220 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 11 Oct 2015 11:01:02 +0200 Subject: Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() uses it to avoid a memory copy. --- Doc/library/binascii.rst | 11 ++++++++--- Lib/base64.py | 3 +-- Lib/test/test_binascii.py | 10 ++++++++++ Misc/NEWS | 3 +++ Modules/binascii.c | 21 +++++++++++++-------- Modules/clinic/binascii.c.h | 17 ++++++++++------- 6 files changed, 45 insertions(+), 20 deletions(-) diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index e3f134b..441aa57 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -52,11 +52,16 @@ The :mod:`binascii` module defines the following functions: than one line may be passed at a time. -.. function:: b2a_base64(data) +.. function:: b2a_base64(data, \*, newline=True) Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char. The length of *data* - should be at most 57 to adhere to the base64 standard. + value is the converted line, including a newline char if *newline* is + true. The length of *data* should be at most 57 to adhere to the + base64 standard. + + + .. versionchanged:: 3.6 + Added the *newline* parameter. .. function:: a2b_qp(data, header=False) diff --git a/Lib/base64.py b/Lib/base64.py index 640f787..eb925cd 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -58,8 +58,7 @@ def b64encode(s, altchars=None): The encoded byte string is returned. """ - # Strip off the trailing newline - encoded = binascii.b2a_base64(s)[:-1] + encoded = binascii.b2a_base64(s, newline=False) if altchars is not None: assert len(altchars) == 2, repr(altchars) return encoded.translate(bytes.maketrans(b'+/', altchars)) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 8367afe..0ab46bc 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -262,6 +262,16 @@ class BinASCIITest(unittest.TestCase): # non-ASCII string self.assertRaises(ValueError, a2b, "\x80") + def test_b2a_base64_newline(self): + # Issue #25357: test newline parameter + b = self.type2test(b'hello') + self.assertEqual(binascii.b2a_base64(b), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=True), + b'aGVsbG8=\n') + self.assertEqual(binascii.b2a_base64(b, newline=False), + b'aGVsbG8=') + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Misc/NEWS b/Misc/NEWS index 802f4b6..e84ad1b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -51,6 +51,9 @@ Core and Builtins Library ------- +- Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). + base64.b64encode() uses it to avoid a memory copy. + - Issue #24164: Objects that need calling ``__new__`` with keyword arguments, can now be pickled using pickle protocols older than protocol version 4. diff --git a/Modules/binascii.c b/Modules/binascii.c index d920d23..3e26a7a 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -528,21 +528,22 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) binascii.b2a_base64 data: Py_buffer - / + * + newline: int(c_default="1") = True Base64-code line of data. [clinic start generated code]*/ static PyObject * -binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) -/*[clinic end generated code: output=3cd61fbee2913285 input=14ec4e47371174a9]*/ +binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) +/*[clinic end generated code: output=19e1dd719a890b50 input=7b2ea6fa38d8924c]*/ { unsigned char *ascii_data, *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; PyObject *rv; - Py_ssize_t bin_len; + Py_ssize_t bin_len, out_len; bin_data = data->buf; bin_len = data->len; @@ -555,9 +556,12 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) } /* We're lazy and allocate too much (fixed up later). - "+3" leaves room for up to two pad characters and a trailing - newline. Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len*2 + 3)) == NULL ) + "+2" leaves room for up to two pad characters. + Note that 'b' gets encoded as 'Yg==\n' (1 in, 5 out). */ + out_len = bin_len*2 + 2; + if (newline) + out_len++; + if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) return NULL; ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); @@ -581,7 +585,8 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data) *ascii_data++ = table_b2a_base64[(leftchar&0xf) << 2]; *ascii_data++ = BASE64_PAD; } - *ascii_data++ = '\n'; /* Append a courtesy newline */ + if (newline) + *ascii_data++ = '\n'; /* Append a courtesy newline */ if (_PyBytes_Resize(&rv, (ascii_data - diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index e348bee..46cfb8e 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -93,26 +93,29 @@ exit: } PyDoc_STRVAR(binascii_b2a_base64__doc__, -"b2a_base64($module, data, /)\n" +"b2a_base64($module, /, data, *, newline=True)\n" "--\n" "\n" "Base64-code line of data."); #define BINASCII_B2A_BASE64_METHODDEF \ - {"b2a_base64", (PyCFunction)binascii_b2a_base64, METH_O, binascii_b2a_base64__doc__}, + {"b2a_base64", (PyCFunction)binascii_b2a_base64, METH_VARARGS|METH_KEYWORDS, binascii_b2a_base64__doc__}, static PyObject * -binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data); +binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline); static PyObject * -binascii_b2a_base64(PyModuleDef *module, PyObject *arg) +binascii_b2a_base64(PyModuleDef *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; + static char *_keywords[] = {"data", "newline", NULL}; Py_buffer data = {NULL, NULL}; + int newline = 1; - if (!PyArg_Parse(arg, "y*:b2a_base64", &data)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|$i:b2a_base64", _keywords, + &data, &newline)) goto exit; - return_value = binascii_b2a_base64_impl(module, &data); + return_value = binascii_b2a_base64_impl(module, &data, newline); exit: /* Cleanup for data */ @@ -516,4 +519,4 @@ exit: return return_value; } -/*[clinic end generated code: output=b1a3cbf7660ebaa5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b15a24350d105251 input=a9049054013a1b77]*/ -- cgit v0.12 From 20d15b5100bc12b21c561ee1ea70d8436f944d98 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 11 Oct 2015 17:52:09 +0300 Subject: Issue #24164: Fixed test_descr: __getnewargs_ex__ now is supported in protocols 2 and 3. --- Lib/test/test_descr.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d096390..d751099 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4738,11 +4738,8 @@ class PicklingTests(unittest.TestCase): return (args, kwargs) obj = C3() for proto in protocols: - if proto >= 4: + if proto >= 2: self._check_reduce(proto, obj, args, kwargs) - elif proto >= 2: - with self.assertRaises(ValueError): - obj.__reduce_ex__(proto) class C4: def __getnewargs_ex__(self): @@ -5061,10 +5058,6 @@ class PicklingTests(unittest.TestCase): kwargs = getattr(cls, 'KWARGS', {}) obj = cls(*cls.ARGS, **kwargs) proto = pickle_copier.proto - if 2 <= proto < 4 and hasattr(cls, '__getnewargs_ex__'): - with self.assertRaises(ValueError): - pickle_copier.dumps(obj, proto) - continue objcopy = pickle_copier.copy(obj) self._assert_is_copy(obj, objcopy) # For test classes that supports this, make sure we didn't go -- cgit v0.12 From 6b1e113f9f142a0e531d0ae3d45cc31341fc9949 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 09:43:50 -0700 Subject: Hoist the deque->maxlen lookup out of the inner-loop. --- Modules/_collectionsmodule.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 52f40b2..8cd22d7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -385,7 +385,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen >= 0); + Py_ssize_t maxlen = deque->maxlen; /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -433,8 +433,10 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (trim) - deque_trim_left(deque); + if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + PyObject *rv = deque_popleft(deque, NULL); + Py_DECREF(rv); + } } return finalize_iterator(it); } @@ -447,7 +449,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) { PyObject *it, *item; PyObject *(*iternext)(PyObject *); - int trim = (deque->maxlen >= 0); + Py_ssize_t maxlen = deque->maxlen; /* Handle case where id(deque) == id(iterable) */ if ((PyObject *)deque == iterable) { @@ -495,8 +497,10 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (trim) - deque_trim_right(deque); + if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + PyObject *rv = deque_pop(deque, NULL); + Py_DECREF(rv); + } } return finalize_iterator(it); } -- cgit v0.12 From d96db09b570bff9e4397530082e33dd3e965a701 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 22:34:48 -0700 Subject: Refactor the deque trim logic to eliminate the two separate trim functions. --- Modules/_collectionsmodule.c | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 8cd22d7..3c8e025 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -276,29 +276,12 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); * the limit. If it has, we get the size back down to the limit by popping an * item off of the opposite end. The methods that can trigger this are append(), * appendleft(), extend(), and extendleft(). + * + * The macro to check whether a deque needs to be trimmed uses a single + * unsigned test that returns true whenever 0 <= maxlen < Py_SIZE(deque). */ -static void -deque_trim_right(dequeobject *deque) -{ - if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { - PyObject *rv = deque_pop(deque, NULL); - assert(rv != NULL); - assert(Py_SIZE(deque) <= deque->maxlen); - Py_DECREF(rv); - } -} - -static void -deque_trim_left(dequeobject *deque) -{ - if (deque->maxlen >= 0 && Py_SIZE(deque) > deque->maxlen) { - PyObject *rv = deque_popleft(deque, NULL); - assert(rv != NULL); - assert(Py_SIZE(deque) <= deque->maxlen); - Py_DECREF(rv); - } -} +#define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) static PyObject * deque_append(dequeobject *deque, PyObject *item) @@ -319,7 +302,10 @@ deque_append(dequeobject *deque, PyObject *item) Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - deque_trim_left(deque); + if (NEEDS_TRIM(deque, deque->maxlen)) { + PyObject *rv = deque_popleft(deque, NULL); + Py_DECREF(rv); + } Py_RETURN_NONE; } @@ -344,7 +330,10 @@ deque_appendleft(dequeobject *deque, PyObject *item) Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - deque_trim_right(deque); + if (NEEDS_TRIM(deque, deque->maxlen)) { + PyObject *rv = deque_pop(deque, NULL); + Py_DECREF(rv); + } Py_RETURN_NONE; } @@ -433,7 +422,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *rv = deque_popleft(deque, NULL); Py_DECREF(rv); } @@ -497,7 +486,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) Py_SIZE(deque)++; deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; - if (maxlen >= 0 && Py_SIZE(deque) > maxlen) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *rv = deque_pop(deque, NULL); Py_DECREF(rv); } -- cgit v0.12 From 965362e92d50d8c4e7e0bfac46f7ac14ee26bbae Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 11 Oct 2015 22:52:54 -0700 Subject: Minor fixup. maxlen is already known. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3c8e025..cef92c0 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -399,7 +399,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (it == NULL) return NULL; - if (deque->maxlen == 0) + if (maxlen == 0) return consume_iterator(it); iternext = *Py_TYPE(it)->tp_iternext; @@ -463,7 +463,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (it == NULL) return NULL; - if (deque->maxlen == 0) + if (maxlen == 0) return consume_iterator(it); iternext = *Py_TYPE(it)->tp_iternext; -- cgit v0.12 From c29e29bed1ea1efc1a0cd3178fac96be4d763ecf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:12:54 +0200 Subject: Relax _PyBytesWriter API Don't require _PyBytesWriter pointer to be a "char *". Same change for _PyBytesWriter_WriteBytes() parameter. For example, binascii uses "unsigned char*". --- Include/bytesobject.h | 14 +++++++------- Objects/bytesobject.c | 15 +++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 2c4c4c4..b7a7c36 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -156,7 +156,7 @@ PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); Return a bytes object. Raise an exception and return NULL on error. */ PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - char *str); + void *str); /* Deallocate memory of a writer (clear its internal buffer). */ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); @@ -164,22 +164,22 @@ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, +PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size); /* Add *size* bytes to the buffer. str is the current pointer inside the buffer. Return the updated current pointer inside the buffer. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - char *str, +PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + void *str, Py_ssize_t size); /* Write bytes. Raise an exception and return NULL on error. */ -PyAPI_FUNC(char*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - char *str, - char *bytes, +PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + void *str, + const void *bytes, Py_ssize_t size); #endif /* Py_LIMITED_API */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a75c54d..4b31271 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3923,8 +3923,8 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #endif } -char* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) +void* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) { Py_ssize_t allocated, pos; @@ -3992,7 +3992,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, char *str, Py_ssize_t size) /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -char* +void* _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) { /* ensure that _PyBytesWriter_Alloc() is only called once */ @@ -4011,7 +4011,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) } PyObject * -_PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) +_PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) { Py_ssize_t pos; PyObject *result; @@ -4033,13 +4033,12 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, char *str) else { result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } - return result; } -char* -_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, char *str, - char *bytes, Py_ssize_t size) +void* +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *str, + const void *bytes, Py_ssize_t size) { str = _PyBytesWriter_Prepare(writer, str, size); if (str == NULL) -- cgit v0.12 From 6c2cdae9e6b664a0b3e204773aeb8954ae1508e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:29:43 +0200 Subject: Writer APIs: use empty string singletons Modify _PyBytesWriter_Finish() and _PyUnicodeWriter_Finish() to return the empty bytes/Unicode string if the string is empty. --- Objects/bytesobject.c | 23 ++++++++++++++--------- Objects/unicodeobject.c | 27 ++++++++++++++++++--------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 4b31271..532051e 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4019,19 +4019,24 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (!writer->use_small_buffer) { + if (pos == 0) { + Py_CLEAR(writer->buffer); + /* Get the empty byte string singleton */ + result = PyBytes_FromStringAndSize(NULL, 0); + } + else if (writer->use_small_buffer) { + result = PyBytes_FromStringAndSize(writer->small_buffer, pos); + } + else { + result = writer->buffer; + writer->buffer = NULL; + if (pos != writer->allocated) { - if (_PyBytes_Resize(&writer->buffer, pos)) { - assert(writer->buffer == NULL); + if (_PyBytes_Resize(&result, pos)) { + assert(result == NULL); return NULL; } } - - result = writer->buffer; - writer->buffer = NULL; - } - else { - result = PyBytes_FromStringAndSize(writer->small_buffer, pos); } return result; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 35df747..4b3746c 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13715,17 +13715,26 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) assert(PyUnicode_GET_LENGTH(str) == writer->pos); return str; } - if (PyUnicode_GET_LENGTH(writer->buffer) != writer->pos) { - PyObject *newbuffer; - newbuffer = resize_compact(writer->buffer, writer->pos); - if (newbuffer == NULL) { - Py_CLEAR(writer->buffer); - return NULL; + if (writer->pos == 0) { + Py_CLEAR(writer->buffer); + + /* Get the empty Unicode string singleton ('') */ + _Py_INCREF_UNICODE_EMPTY(); + str = unicode_empty; + } + else { + str = writer->buffer; + writer->buffer = NULL; + + if (PyUnicode_GET_LENGTH(str) != writer->pos) { + PyObject *str2; + str2 = resize_compact(str, writer->pos); + if (str2 == NULL) + return NULL; + str = str2; } - writer->buffer = newbuffer; } - str = writer->buffer; - writer->buffer = NULL; + assert(_PyUnicode_CheckConsistency(str, 1)); return unicode_result_ready(str); } -- cgit v0.12 From e9aa5950bb6ae7eab553e915402eddcb610565e4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 13:57:47 +0200 Subject: Fix compilation error in _PyBytesWriter_WriteBytes() on Windows --- Objects/bytesobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 532051e..e7ab503 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4042,9 +4042,11 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) } void* -_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *str, +_PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *ptr, const void *bytes, Py_ssize_t size) { + char *str = (char *)ptr; + str = _PyBytesWriter_Prepare(writer, str, size); if (str == NULL) return NULL; -- cgit v0.12 From d65e4f4eea278357e5aaee9f510922ef83e04143 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 14:38:24 +0200 Subject: Issue #24164: Fix test_pyclbr Ignore pickle.partial symbol which comes from functools.partial. --- Lib/test/test_pyclbr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index cab430b..48bd725 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -156,7 +156,7 @@ class PyclbrTest(TestCase): # These were once about the 10 longest modules cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator cm('cgi', ignore=('log',)) # set with = in module - cm('pickle') + cm('pickle', ignore=('partial',)) cm('aifc', ignore=('openfp', '_aifc_params')) # set with = in module cm('sre_parse', ignore=('dump', 'groups')) # from sre_constants import *; property cm('pdb') -- cgit v0.12 From 358af1352689fc10c81690a193ff5414f5f930af Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Oct 2015 22:36:57 +0200 Subject: Issue #25353: Optimize unicode escape and raw unicode escape encoders to use the new _PyBytesWriter API. --- Modules/_pickle.c | 44 ++++++++++--------- Objects/unicodeobject.c | 113 +++++++++++++++++++++++++++++------------------- 2 files changed, 93 insertions(+), 64 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index abaf4e5..341ac0d 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2110,38 +2110,35 @@ save_bytes(PicklerObject *self, PyObject *obj) static PyObject * raw_unicode_escape(PyObject *obj) { - PyObject *repr; char *p; Py_ssize_t i, size; - size_t expandsize; void *data; unsigned int kind; + _PyBytesWriter writer; if (PyUnicode_READY(obj)) return NULL; + _PyBytesWriter_Init(&writer); + size = PyUnicode_GET_LENGTH(obj); data = PyUnicode_DATA(obj); kind = PyUnicode_KIND(obj); - if (kind == PyUnicode_4BYTE_KIND) - expandsize = 10; - else - expandsize = 6; - if ((size_t)size > (size_t)PY_SSIZE_T_MAX / expandsize) - return PyErr_NoMemory(); - repr = PyBytes_FromStringAndSize(NULL, expandsize * size); - if (repr == NULL) - return NULL; - if (size == 0) - return repr; - assert(Py_REFCNT(repr) == 1); + p = _PyBytesWriter_Alloc(&writer, size); + if (p == NULL) + goto error; + writer.overallocate = 1; - p = PyBytes_AS_STRING(repr); for (i=0; i < size; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0xf]; @@ -2153,8 +2150,13 @@ raw_unicode_escape(PyObject *obj) *p++ = Py_hexdigits[(ch >> 4) & 0xf]; *p++ = Py_hexdigits[ch & 15]; } - /* Map 16-bit characters to '\uxxxx' */ + /* Map 16-bit characters, '\\' and '\n' to '\uxxxx' */ else if (ch >= 256 || ch == '\\' || ch == '\n') { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; @@ -2166,10 +2168,12 @@ raw_unicode_escape(PyObject *obj) else *p++ = (char) ch; } - size = p - PyBytes_AS_STRING(repr); - if (_PyBytes_Resize(&repr, size) < 0) - return NULL; - return repr; + + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } static int diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4b3746c..f5044c8 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6052,11 +6052,10 @@ PyObject * PyUnicode_AsUnicodeEscapeString(PyObject *unicode) { Py_ssize_t i, len; - PyObject *repr; char *p; int kind; void *data; - Py_ssize_t expandsize = 0; + _PyBytesWriter writer; /* Initial allocation is based on the longest-possible character escape. @@ -6072,35 +6071,28 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) } if (PyUnicode_READY(unicode) == -1) return NULL; + + _PyBytesWriter_Init(&writer); + len = PyUnicode_GET_LENGTH(unicode); kind = PyUnicode_KIND(unicode); data = PyUnicode_DATA(unicode); - switch (kind) { - case PyUnicode_1BYTE_KIND: expandsize = 4; break; - case PyUnicode_2BYTE_KIND: expandsize = 6; break; - case PyUnicode_4BYTE_KIND: expandsize = 10; break; - } - - if (len == 0) - return PyBytes_FromStringAndSize(NULL, 0); - - if (len > (PY_SSIZE_T_MAX - 2 - 1) / expandsize) - return PyErr_NoMemory(); - repr = PyBytes_FromStringAndSize(NULL, - 2 - + expandsize*len - + 1); - if (repr == NULL) - return NULL; - - p = PyBytes_AS_STRING(repr); + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) + goto error; + writer.overallocate = 1; for (i = 0; i < len; i++) { Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Escape backslashes */ if (ch == '\\') { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = (char) ch; continue; @@ -6109,6 +6101,11 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map 21-bit characters to '\U00xxxxxx' */ else if (ch >= 0x10000) { assert(ch <= MAX_UNICODE); + + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0x0000000F]; @@ -6124,6 +6121,10 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map 16-bit characters to '\uxxxx' */ if (ch >= 256) { + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0x000F]; @@ -6134,20 +6135,37 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) /* Map special whitespace to '\t', \n', '\r' */ else if (ch == '\t') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 't'; } else if (ch == '\n') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'n'; } else if (ch == '\r') { + p = _PyBytesWriter_Prepare(&writer, p, 2-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'r'; } /* Map non-printable US ASCII to '\xhh' */ else if (ch < ' ' || ch >= 0x7F) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 4-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'x'; *p++ = Py_hexdigits[(ch >> 4) & 0x000F]; @@ -6159,10 +6177,11 @@ PyUnicode_AsUnicodeEscapeString(PyObject *unicode) *p++ = (char) ch; } - assert(p - PyBytes_AS_STRING(repr) > 0); - if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0) - return NULL; - return repr; + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * @@ -6291,13 +6310,12 @@ PyUnicode_DecodeRawUnicodeEscape(const char *s, PyObject * PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) { - PyObject *repr; char *p; - char *q; - Py_ssize_t expandsize, pos; + Py_ssize_t pos; int kind; void *data; Py_ssize_t len; + _PyBytesWriter writer; if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); @@ -6305,28 +6323,29 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) } if (PyUnicode_READY(unicode) == -1) return NULL; + + _PyBytesWriter_Init(&writer); + kind = PyUnicode_KIND(unicode); data = PyUnicode_DATA(unicode); len = PyUnicode_GET_LENGTH(unicode); - /* 4 byte characters can take up 10 bytes, 2 byte characters can take up 6 - bytes, and 1 byte characters 4. */ - expandsize = kind * 2 + 2; - if (len > PY_SSIZE_T_MAX / expandsize) - return PyErr_NoMemory(); - - repr = PyBytes_FromStringAndSize(NULL, expandsize * len); - if (repr == NULL) - return NULL; - if (len == 0) - return repr; + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) + goto error; + writer.overallocate = 1; - p = q = PyBytes_AS_STRING(repr); for (pos = 0; pos < len; pos++) { Py_UCS4 ch = PyUnicode_READ(kind, data, pos); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { assert(ch <= MAX_UNICODE); + + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 10-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'U'; *p++ = Py_hexdigits[(ch >> 28) & 0xf]; @@ -6340,6 +6359,11 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) } /* Map 16-bit characters to '\uxxxx' */ else if (ch >= 256) { + /* -1: substract 1 preallocated byte */ + p = _PyBytesWriter_Prepare(&writer, p, 6-1); + if (p == NULL) + goto error; + *p++ = '\\'; *p++ = 'u'; *p++ = Py_hexdigits[(ch >> 12) & 0xf]; @@ -6352,10 +6376,11 @@ PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) *p++ = (char) ch; } - assert(p > q); - if (_PyBytes_Resize(&repr, p - q) < 0) - return NULL; - return repr; + return _PyBytesWriter_Finish(&writer, p); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * -- cgit v0.12 From eaaaf136d20e1dd5160c2083fb81313d0818ea5d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 13 Oct 2015 10:51:47 +0200 Subject: Issue #25384: Use _PyBytesWriter API in binascii This API avoids a final call to _PyBytes_Resize() for output smaller than 512 bytes. Small optimization: disable overallocation in binascii.rledecode_hqx() for the last write. --- Modules/binascii.c | 194 +++++++++++++++++++++++------------------------------ 1 file changed, 83 insertions(+), 111 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index 3e26a7a..dfa5535 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -346,9 +346,10 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; - Py_ssize_t bin_len; + Py_ssize_t bin_len, out_len; + _PyBytesWriter writer; + _PyBytesWriter_Init(&writer); bin_data = data->buf; bin_len = data->len; if ( bin_len > 45 ) { @@ -358,9 +359,10 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) } /* We're lazy and allocate to much (fixed up later) */ - if ( (rv=PyBytes_FromStringAndSize(NULL, 2 + (bin_len+2)/3*4)) == NULL ) + out_len = 2 + (bin_len + 2) / 3 * 4; + ascii_data = _PyBytesWriter_Alloc(&writer, out_len); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); /* Store the length */ *ascii_data++ = ' ' + (bin_len & 077); @@ -382,12 +384,7 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) } *ascii_data++ = '\n'; /* Append a courtesy newline */ - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, ascii_data); } @@ -433,9 +430,9 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t ascii_len, bin_len; int quad_pos = 0; + _PyBytesWriter writer; ascii_data = data->buf; ascii_len = data->len; @@ -447,11 +444,12 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) bin_len = ((ascii_len+3)/4)*3; /* Upper bound, corrected later */ + _PyBytesWriter_Init(&writer); + /* Allocate the buffer */ - if ( (rv=PyBytes_FromStringAndSize(NULL, bin_len)) == NULL ) + bin_data = _PyBytesWriter_Alloc(&writer, bin_len); + if (bin_data == NULL) return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); - bin_len = 0; for( ; ascii_len > 0; ascii_len--, ascii_data++) { this_ch = *ascii_data; @@ -496,31 +494,17 @@ binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) if ( leftbits >= 8 ) { leftbits -= 8; *bin_data++ = (leftchar >> leftbits) & 0xff; - bin_len++; leftchar &= ((1 << leftbits) - 1); } } if (leftbits != 0) { PyErr_SetString(Error, "Incorrect padding"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } - /* And set string size correctly. If the result string is empty - ** (because the input was all invalid) return the shared empty - ** string instead; _PyBytes_Resize() won't do this for us. - */ - if (bin_len > 0) { - if (_PyBytes_Resize(&rv, bin_len) < 0) { - Py_CLEAR(rv); - } - } - else { - Py_DECREF(rv); - rv = PyBytes_FromStringAndSize("", 0); - } - return rv; + return _PyBytesWriter_Finish(&writer, bin_data); } @@ -542,11 +526,12 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; Py_ssize_t bin_len, out_len; + _PyBytesWriter writer; bin_data = data->buf; bin_len = data->len; + _PyBytesWriter_Init(&writer); assert(bin_len >= 0); @@ -561,9 +546,9 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) out_len = bin_len*2 + 2; if (newline) out_len++; - if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) + ascii_data = _PyBytesWriter_Alloc(&writer, out_len); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; bin_len > 0 ; bin_len--, bin_data++ ) { /* Shift the data into our buffer */ @@ -588,12 +573,7 @@ binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) if (newline) *ascii_data++ = '\n'; /* Append a courtesy newline */ - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, ascii_data); } /*[clinic input] @@ -613,12 +593,14 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; - PyObject *rv; + PyObject *res; Py_ssize_t len; int done = 0; + _PyBytesWriter writer; ascii_data = data->buf; len = data->len; + _PyBytesWriter_Init(&writer); assert(len >= 0); @@ -628,9 +610,9 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) /* Allocate a string that is too big (fixed later) Add two to the initial length to prevent interning which would preclude subsequent resizing. */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len+2)) == NULL ) + bin_data = _PyBytesWriter_Alloc(&writer, len + 2); + if (bin_data == NULL) return NULL; - bin_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; len > 0 ; len--, ascii_data++ ) { /* Get the byte and look it up */ @@ -639,7 +621,7 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) continue; if ( this_ch == FAIL ) { PyErr_SetString(Error, "Illegal char"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } if ( this_ch == DONE ) { @@ -661,21 +643,14 @@ binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) if ( leftbits && !done ) { PyErr_SetString(Incomplete, "String has incomplete number of bytes"); - Py_DECREF(rv); + _PyBytesWriter_Dealloc(&writer); return NULL; } - if (_PyBytes_Resize(&rv, - (bin_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - if (rv) { - PyObject *rrv = Py_BuildValue("Oi", rv, done); - Py_DECREF(rv); - return rrv; - } - return NULL; + res = _PyBytesWriter_Finish(&writer, bin_data); + if (res == NULL) + return NULL; + return Py_BuildValue("Ni", res, done); } @@ -693,10 +668,11 @@ binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=0905da344dbf0648 input=e1f1712447a82b09]*/ { unsigned char *in_data, *out_data; - PyObject *rv; unsigned char ch; Py_ssize_t in, inend, len; + _PyBytesWriter writer; + _PyBytesWriter_Init(&writer); in_data = data->buf; len = data->len; @@ -706,9 +682,9 @@ binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Worst case: output is twice as big as input (fixed later) */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len*2+2)) == NULL ) + out_data = _PyBytesWriter_Alloc(&writer, len * 2 + 2); + if (out_data == NULL) return NULL; - out_data = (unsigned char *)PyBytes_AS_STRING(rv); for( in=0; inbuf; len = data->len; + _PyBytesWriter_Init(&writer); assert(len >= 0); @@ -772,9 +745,9 @@ binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer that is at least large enough */ - if ( (rv=PyBytes_FromStringAndSize(NULL, len*2+2)) == NULL ) + ascii_data = _PyBytesWriter_Alloc(&writer, len * 2 + 2); + if (ascii_data == NULL) return NULL; - ascii_data = (unsigned char *)PyBytes_AS_STRING(rv); for( ; len > 0 ; len--, bin_data++ ) { /* Shift into our buffer, and output any 6bits ready */ @@ -791,12 +764,8 @@ binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) leftchar <<= (6-leftbits); *ascii_data++ = table_b2a_hqx[leftchar & 0x3f]; } - if (_PyBytes_Resize(&rv, - (ascii_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + + return _PyBytesWriter_Finish(&writer, ascii_data); } @@ -815,11 +784,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) { unsigned char *in_data, *out_data; unsigned char in_byte, in_repeat; - PyObject *rv; Py_ssize_t in_len, out_len, out_len_left; + _PyBytesWriter writer; in_data = data->buf; in_len = data->len; + _PyBytesWriter_Init(&writer); assert(in_len >= 0); @@ -830,45 +800,49 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len*2; - if ( (rv=PyBytes_FromStringAndSize(NULL, out_len)) == NULL ) + out_len = in_len * 2; + out_data = _PyBytesWriter_Alloc(&writer, out_len); + if (out_data == NULL) return NULL; - out_len_left = out_len; - out_data = (unsigned char *)PyBytes_AS_STRING(rv); + + /* Use overallocation */ + writer.overallocate = 1; + out_len_left = writer.allocated; /* ** We need two macros here to get/put bytes and handle ** end-of-buffer for input and output strings. */ -#define INBYTE(b) \ - do { \ - if ( --in_len < 0 ) { \ - PyErr_SetString(Incomplete, ""); \ - Py_DECREF(rv); \ - return NULL; \ - } \ - b = *in_data++; \ +#define INBYTE(b) \ + do { \ + if ( --in_len < 0 ) { \ + PyErr_SetString(Incomplete, ""); \ + goto error; \ + } \ + b = *in_data++; \ } while(0) -#define OUTBYTE(b) \ - do { \ - if ( --out_len_left < 0 ) { \ - if ( out_len > PY_SSIZE_T_MAX / 2) return PyErr_NoMemory(); \ - if (_PyBytes_Resize(&rv, 2*out_len) < 0) \ - { Py_XDECREF(rv); return NULL; } \ - out_data = (unsigned char *)PyBytes_AS_STRING(rv) \ - + out_len; \ - out_len_left = out_len-1; \ - out_len = out_len * 2; \ - } \ - *out_data++ = b; \ +#define OUTBYTE(b) \ + do { \ + if ( --out_len_left < 0 ) { \ + if (in_len <= 0) { \ + /* We are done after this write, no need to \ + overallocate the buffer anymore */ \ + writer.overallocate = 0; \ + } \ + out_data = _PyBytesWriter_Prepare(&writer, out_data, 1); \ + if (out_data == NULL) \ + goto error; \ + out_len_left = writer.allocated; \ + } \ + *out_data++ = b; \ } while(0) - /* - ** Handle first byte separately (since we have to get angry - ** in case of an orphaned RLE code). - */ - INBYTE(in_byte); + /* + ** Handle first byte separately (since we have to get angry + ** in case of an orphaned RLE code). + */ + INBYTE(in_byte); if (in_byte == RUNCHAR) { INBYTE(in_repeat); @@ -877,8 +851,7 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) ** of the string only). This is a programmer error. */ PyErr_SetString(Error, "Orphaned RLE code at start"); - Py_DECREF(rv); - return NULL; + goto error; } OUTBYTE(RUNCHAR); } else { @@ -904,12 +877,11 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) OUTBYTE(in_byte); } } - if (_PyBytes_Resize(&rv, - (out_data - - (unsigned char *)PyBytes_AS_STRING(rv))) < 0) { - Py_CLEAR(rv); - } - return rv; + return _PyBytesWriter_Finish(&writer, out_data); + +error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } -- cgit v0.12 From dbc517c2dd273f70d2608fd514477fbac459fcab Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 13 Oct 2015 21:20:14 +0300 Subject: Issue #25382: pickletools.dis() now outputs implicit memo index for the MEMOIZE opcode. --- Lib/pickletools.py | 1 + Misc/NEWS | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index cf5df41..43dedb3 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2440,6 +2440,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): if opcode.name in ("PUT", "BINPUT", "LONG_BINPUT", "MEMOIZE"): if opcode.name == "MEMOIZE": memo_idx = len(memo) + markmsg = "(as %d)" % memo_idx else: assert arg is not None memo_idx = arg diff --git a/Misc/NEWS b/Misc/NEWS index 142360e..2700ad3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -56,6 +56,9 @@ Core and Builtins Library ------- +- Issue #25382: pickletools.dis() now outputs implicit memo index for the + MEMOIZE opcode. + - Issue #25357: Add an optional newline paramer to binascii.b2a_base64(). base64.b64encode() uses it to avoid a memory copy. -- cgit v0.12 From b6d84832bfa1f453d3c73d6db0982aee0e6d04fe Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 13 Oct 2015 21:26:35 +0300 Subject: Issue #24164: Document changes to __getnewargs__ and __getnewargs_ex__. --- Doc/library/pickle.rst | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f862065..2aab909 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -488,7 +488,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 4 and newer, classes that implements the + In protocols 2 and newer, classes that implements the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -500,15 +500,22 @@ methods: class requires keyword-only arguments. Otherwise, it is recommended for compatibility to implement :meth:`__getnewargs__`. + .. versionchanged:: 3.6 + :meth:`__getnewargs_ex__` is now used in protocols 2 and 3. + .. method:: object.__getnewargs__() - This method serve a similar purpose as :meth:`__getnewargs_ex__` but - for protocols 2 and newer. It must return a tuple of arguments ``args`` - which will be passed to the :meth:`__new__` method upon unpickling. + This method serve a similar purpose as :meth:`__getnewargs_ex__`, but + supports only positional arguments. It must return a tuple of arguments + ``args`` which will be passed to the :meth:`__new__` method upon unpickling. + + :meth:`__getnewargs__` will not be called if :meth:`__getnewargs_ex__` is + defined. - In protocols 4 and newer, :meth:`__getnewargs__` will not be called if - :meth:`__getnewargs_ex__` is defined. + .. versionchanged:: 3.6 + Before Python 3.6, :meth:`__getnewargs__` was called instead of + :meth:`__getnewargs_ex__` in protocols 2 and 3. .. method:: object.__getstate__() -- cgit v0.12 From 03dab786b2f504791ac46a9f9b9db82e634efd05 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 00:21:35 +0200 Subject: Rewrite PyBytes_FromFormatV() using _PyBytesWriter API * Add much more unit tests on PyBytes_FromFormatV() * Remove the first loop to compute the length of the output string * Use _PyBytesWriter to handle the bytes buffer, use overallocation * Cleanup the code to make simpler and easier to review --- Lib/test/test_bytes.py | 92 ++++++++++++-- Objects/bytesobject.c | 336 ++++++++++++++++++++++++------------------------- 2 files changed, 245 insertions(+), 183 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 53a80f4..5fe193e 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -783,25 +783,93 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): test.support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int, c_char_p + _testcapi = test.support.import_module('_testcapi') + from ctypes import pythonapi, py_object + from ctypes import ( + c_int, c_uint, + c_long, c_ulong, + c_size_t, c_ssize_t, + c_char_p) + PyBytes_FromFormat = pythonapi.PyBytes_FromFormat PyBytes_FromFormat.restype = py_object + # basic tests self.assertEqual(PyBytes_FromFormat(b'format'), b'format') - + self.assertEqual(PyBytes_FromFormat(b'Hello %s !', b'world'), + b'Hello world !') + + # test formatters + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(0)), + b'c=\0') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(ord('@'))), + b'c=@') + self.assertEqual(PyBytes_FromFormat(b'c=%c', c_int(255)), + b'c=\xff') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(1), c_long(2), + c_size_t(3)), + b'd=1 ld=2 zd=3') + self.assertEqual(PyBytes_FromFormat(b'd=%d ld=%ld zd=%zd', + c_int(-1), c_long(-2), + c_size_t(-3)), + b'd=-1 ld=-2 zd=-3') + self.assertEqual(PyBytes_FromFormat(b'u=%u lu=%lu zu=%zu', + c_uint(123), c_ulong(456), + c_size_t(789)), + b'u=123 lu=456 zu=789') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(123)), + b'i=123') + self.assertEqual(PyBytes_FromFormat(b'i=%i', c_int(-123)), + b'i=-123') + self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), + b'x=abc') + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', + c_char_p(0xabcdef)), + b'ptr=0xabcdef') + self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), + b's=cstr') + + # test minimum and maximum integer values + size_max = c_size_t(-1).value + for formatstr, ctypes_type, value, py_formatter in ( + (b'%d', c_int, _testcapi.INT_MIN, str), + (b'%d', c_int, _testcapi.INT_MAX, str), + (b'%ld', c_long, _testcapi.LONG_MIN, str), + (b'%ld', c_long, _testcapi.LONG_MAX, str), + (b'%lu', c_ulong, _testcapi.ULONG_MAX, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), + (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), + (b'%zu', c_size_t, size_max, str), + (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + ): + self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), + py_formatter(value).encode('ascii')), + + # width and precision (width is currently ignored) + self.assertEqual(PyBytes_FromFormat(b'%5s', b'a'), + b'a') + self.assertEqual(PyBytes_FromFormat(b'%.3s', b'abcdef'), + b'abc') + + # '%%' formatter + self.assertEqual(PyBytes_FromFormat(b'%%'), + b'%') + self.assertEqual(PyBytes_FromFormat(b'[%%]'), + b'[%]') + self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), + b'%_') + self.assertEqual(PyBytes_FromFormat(b'%%s'), + b'%s') + + # Invalid formats and partial formatting self.assertEqual(PyBytes_FromFormat(b'%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%'), b'%') - self.assertEqual(PyBytes_FromFormat(b'%%s'), b'%s') - self.assertEqual(PyBytes_FromFormat(b'[%%]'), b'[%]') - self.assertEqual(PyBytes_FromFormat(b'%%%c', c_int(ord('_'))), b'%_') - - self.assertEqual(PyBytes_FromFormat(b'c:%c', c_int(255)), - b'c:\xff') - self.assertEqual(PyBytes_FromFormat(b's:%s', c_char_p(b'cstr')), - b's:cstr') + self.assertEqual(PyBytes_FromFormat(b'x=%i y=%', c_int(2), c_int(3)), + b'x=2 y=%') - # Issue #19969 + # Issue #19969: %c must raise OverflowError for values + # not in the range [0; 255] self.assertRaises(OverflowError, PyBytes_FromFormat, b'%c', c_int(-1)) self.assertRaises(OverflowError, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e7ab503..189673c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -174,190 +174,184 @@ PyBytes_FromString(const char *str) PyObject * PyBytes_FromFormatV(const char *format, va_list vargs) { - va_list count; - Py_ssize_t n = 0; - const char* f; char *s; - PyObject* string; + const char *f; + const char *p; + Py_ssize_t prec; + int longflag; + int size_tflag; + /* Longest 64-bit formatted numbers: + - "18446744073709551615\0" (21 bytes) + - "-9223372036854775808\0" (21 bytes) + Decimal takes the most space (it isn't enough for octal.) + + Longest 64-bit pointer representation: + "0xffffffffffffffff\0" (19 bytes). */ + char buffer[21]; + _PyBytesWriter writer; - Py_VA_COPY(count, vargs); - /* step 1: figure out how large a buffer we need */ - for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f; - while (*++f && *f != '%' && !Py_ISALPHA(*f)) - ; - - /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since - * they don't affect the amount of space we reserve. - */ - if ((*f == 'l' || *f == 'z') && - (f[1] == 'd' || f[1] == 'u')) - ++f; - - switch (*f) { - case 'c': - { - int c = va_arg(count, int); - if (c < 0 || c > 255) { - PyErr_SetString(PyExc_OverflowError, - "PyBytes_FromFormatV(): %c format " - "expects an integer in range [0; 255]"); - return NULL; - } - n++; - break; - } - case '%': - n++; - break; - case 'd': case 'u': case 'i': case 'x': - (void) va_arg(count, int); - /* 20 bytes is enough to hold a 64-bit - integer. Decimal takes the most space. - This isn't enough for octal. */ - n += 20; - break; - case 's': - s = va_arg(count, char*); - n += strlen(s); - break; - case 'p': - (void) va_arg(count, int); - /* maximum 64-bit pointer representation: - * 0xffffffffffffffff - * so 19 characters is enough. - * XXX I count 18 -- what's the extra for? - */ - n += 19; - break; - default: - /* if we stumble upon an unknown - formatting code, copy the rest of - the format string to the output - string. (we cannot just skip the - code, since there's no way to know - what's in the argument list) */ - n += strlen(p); - goto expand; - } - } else - n++; - } - expand: - /* step 2: fill the buffer */ - /* Since we've analyzed how much space we need for the worst case, - use sprintf directly instead of the slower PyOS_snprintf. */ - string = PyBytes_FromStringAndSize(NULL, n); - if (!string) + _PyBytesWriter_Init(&writer); + + s = _PyBytesWriter_Alloc(&writer, strlen(format)); + if (s == NULL) return NULL; + writer.overallocate = 1; - s = PyBytes_AsString(string); +#define WRITE_BYTES(str) \ + do { \ + s = _PyBytesWriter_WriteBytes(&writer, s, (str), strlen(str)); \ + if (s == NULL) \ + goto error; \ + } while (0) for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f++; - Py_ssize_t i; - int longflag = 0; - int size_tflag = 0; - /* parse the width.precision part (we're only - interested in the precision value, if any) */ - n = 0; - while (Py_ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - if (*f == '.') { - f++; - n = 0; - while (Py_ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - } - while (*f && *f != '%' && !Py_ISALPHA(*f)) - f++; - /* handle the long flag, but only for %ld and %lu. - others can be added when necessary. */ - if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { - longflag = 1; - ++f; + if (*f != '%') { + *s++ = *f; + continue; + } + + p = f++; + + /* ignore the width (ex: 10 in "%10s") */ + while (Py_ISDIGIT(*f)) + f++; + + /* parse the precision (ex: 10 in "%.10s") */ + prec = 0; + if (*f == '.') { + f++; + for (; Py_ISDIGIT(*f); f++) { + prec = (prec * 10) + (*f - '0'); } - /* handle the size_t flag. */ - if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { - size_tflag = 1; - ++f; + } + + while (*f && *f != '%' && !Py_ISALPHA(*f)) + f++; + + /* handle the long flag ('l'), but only for %ld and %lu. + others can be added when necessary. */ + longflag = 0; + if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { + longflag = 1; + ++f; + } + + /* handle the size_t flag ('z'). */ + size_tflag = 0; + if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + size_tflag = 1; + ++f; + } + + /* substract bytes preallocated for the format string + (ex: 2 for "%s") */ + writer.min_size -= (f - p + 1); + + switch (*f) { + case 'c': + { + int c = va_arg(vargs, int); + if (c < 0 || c > 255) { + PyErr_SetString(PyExc_OverflowError, + "PyBytes_FromFormatV(): %c format " + "expects an integer in range [0; 255]"); + goto error; } + writer.min_size++; + *s++ = (unsigned char)c; + break; + } - switch (*f) { - case 'c': - { - int c = va_arg(vargs, int); - /* c has been checked for overflow in the first step */ - *s++ = (unsigned char)c; - break; + case 'd': + if (longflag) + sprintf(buffer, "%ld", va_arg(vargs, long)); + else if (size_tflag) + sprintf(buffer, "%" PY_FORMAT_SIZE_T "d", + va_arg(vargs, Py_ssize_t)); + else + sprintf(buffer, "%d", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'u': + if (longflag) + sprintf(buffer, "%lu", + va_arg(vargs, unsigned long)); + else if (size_tflag) + sprintf(buffer, "%" PY_FORMAT_SIZE_T "u", + va_arg(vargs, size_t)); + else + sprintf(buffer, "%u", + va_arg(vargs, unsigned int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'i': + sprintf(buffer, "%i", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 'x': + sprintf(buffer, "%x", va_arg(vargs, int)); + assert(strlen(buffer) < sizeof(buffer)); + WRITE_BYTES(buffer); + break; + + case 's': + { + Py_ssize_t i; + + p = va_arg(vargs, char*); + i = strlen(p); + if (prec > 0 && i > prec) + i = prec; + s = _PyBytesWriter_WriteBytes(&writer, s, p, i); + if (s == NULL) + goto error; + break; + } + + case 'p': + sprintf(buffer, "%p", va_arg(vargs, void*)); + assert(strlen(buffer) < sizeof(buffer)); + /* %p is ill-defined: ensure leading 0x. */ + if (buffer[1] == 'X') + buffer[1] = 'x'; + else if (buffer[1] != 'x') { + memmove(buffer+2, buffer, strlen(buffer)+1); + buffer[0] = '0'; + buffer[1] = 'x'; } - case 'd': - if (longflag) - sprintf(s, "%ld", va_arg(vargs, long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "d", - va_arg(vargs, Py_ssize_t)); - else - sprintf(s, "%d", va_arg(vargs, int)); - s += strlen(s); - break; - case 'u': - if (longflag) - sprintf(s, "%lu", - va_arg(vargs, unsigned long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "u", - va_arg(vargs, size_t)); - else - sprintf(s, "%u", - va_arg(vargs, unsigned int)); - s += strlen(s); - break; - case 'i': - sprintf(s, "%i", va_arg(vargs, int)); - s += strlen(s); - break; - case 'x': - sprintf(s, "%x", va_arg(vargs, int)); - s += strlen(s); - break; - case 's': - p = va_arg(vargs, char*); - i = strlen(p); - if (n > 0 && i > n) - i = n; - Py_MEMCPY(s, p, i); - s += i; - break; - case 'p': - sprintf(s, "%p", va_arg(vargs, void*)); - /* %p is ill-defined: ensure leading 0x. */ - if (s[1] == 'X') - s[1] = 'x'; - else if (s[1] != 'x') { - memmove(s+2, s, strlen(s)+1); - s[0] = '0'; - s[1] = 'x'; - } - s += strlen(s); - break; - case '%': - *s++ = '%'; - break; - default: - strcpy(s, p); - s += strlen(s); - goto end; + WRITE_BYTES(buffer); + break; + + case '%': + writer.min_size++; + *s++ = '%'; + break; + + default: + if (*f == 0) { + /* fix min_size if we reached the end of the format string */ + writer.min_size++; } - } else - *s++ = *f; + + /* invalid format string: copy unformatted string and exit */ + WRITE_BYTES(p); + return _PyBytesWriter_Finish(&writer, s); + } } - end: - _PyBytes_Resize(&string, s - PyBytes_AS_STRING(string)); - return string; +#undef WRITE_BYTES + + return _PyBytesWriter_Finish(&writer, s); + + error: + _PyBytesWriter_Dealloc(&writer); + return NULL; } PyObject * -- cgit v0.12 From 7ab986dd84b43742af9b504e0c7acfef8f231ab3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 02:55:12 +0200 Subject: Fix test_bytes on Windows On Windows, sprintf("%p", 0xabcdef) formats hexadecimal in uppercase and pad to 16 characters (on 64-bit system) with zeros. --- Lib/test/test_bytes.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 5fe193e..947d959 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -782,7 +782,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): # Test PyBytes_FromFormat() def test_from_format(self): - test.support.import_module('ctypes') + ctypes = test.support.import_module('ctypes') _testcapi = test.support.import_module('_testcapi') from ctypes import pythonapi, py_object from ctypes import ( @@ -825,9 +825,12 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') - self.assertEqual(PyBytes_FromFormat(b'ptr=%p', - c_char_p(0xabcdef)), - b'ptr=0xabcdef') + ptr = 0xabcdef + expected = [b'ptr=%#x' % ptr] + win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) + expected.append((win_format % ptr).encode('ascii')) + self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + expected) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') -- cgit v0.12 From 199c9a6f4bf6ff6fe04c4c4ca17c24bc258079f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:47:23 +0200 Subject: Fix long_format_binary() Issue #25399: Fix long_format_binary(), allocate bytes for the bytes writer. --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 00f5d95..759116a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1836,7 +1836,7 @@ long_format_binary(PyObject *aa, int base, int alternate, kind = writer->kind; v = NULL; } - else if (writer) { + else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); if (*bytes_str == NULL) return -1; -- cgit v0.12 From 661aaccf9def380540cc1d440761159a414094d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:41:48 +0200 Subject: Add use_bytearray attribute to _PyBytesWriter Issue #25399: Add a new use_bytearray attribute to _PyBytesWriter to use a bytearray buffer, instead of using a bytes object. --- Include/bytesobject.h | 12 ++++--- Objects/bytesobject.c | 93 +++++++++++++++++++++++++++++++++++---------------- 2 files changed, 73 insertions(+), 32 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index b7a7c36..fbb6322 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -128,17 +128,21 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, A _PyBytesWriter variable must be declared at the end of variables in a function to optimize the memory allocation on the stack. */ typedef struct { - /* bytes object */ + /* bytes, bytearray or NULL (when the small buffer is used) */ PyObject *buffer; - /* Number of allocated size */ + /* Number of allocated size. */ Py_ssize_t allocated; /* Minimum number of allocated bytes, incremented by _PyBytesWriter_Prepare() */ Py_ssize_t min_size; - /* If non-zero, overallocate the buffer (default: 0). */ + /* If non-zero, use a bytearray instead of a bytes object for buffer. */ + int use_bytearray; + + /* If non-zero, overallocate the buffer (default: 0). + This flag must be zero if use_bytearray is non-zero. */ int overallocate; /* Stack buffer */ @@ -153,7 +157,7 @@ typedef struct { PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); /* Get the buffer content and reset the writer. - Return a bytes object. + Return a bytes object, or a bytearray object if use_bytearray is non-zero. Raise an exception and return NULL on error. */ PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str); diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 189673c..a1f2958 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3852,11 +3852,8 @@ bytes_iter(PyObject *seq) void _PyBytesWriter_Init(_PyBytesWriter *writer) { - writer->buffer = NULL; - writer->allocated = 0; - writer->min_size = 0; - writer->overallocate = 0; - writer->use_small_buffer = 0; + /* Set all attributes before small_buffer to 0 */ + memset(writer, 0, offsetof(_PyBytesWriter, small_buffer)); #ifdef Py_DEBUG memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer)); #endif @@ -3871,13 +3868,17 @@ _PyBytesWriter_Dealloc(_PyBytesWriter *writer) Py_LOCAL_INLINE(char*) _PyBytesWriter_AsString(_PyBytesWriter *writer) { - if (!writer->use_small_buffer) { + if (writer->use_small_buffer) { + assert(writer->buffer == NULL); + return writer->small_buffer; + } + else if (writer->use_bytearray) { assert(writer->buffer != NULL); - return PyBytes_AS_STRING(writer->buffer); + return PyByteArray_AS_STRING(writer->buffer); } else { - assert(writer->buffer == NULL); - return writer->small_buffer; + assert(writer->buffer != NULL); + return PyBytes_AS_STRING(writer->buffer); } } @@ -3897,18 +3898,28 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) #ifdef Py_DEBUG char *start, *end; - if (!writer->use_small_buffer) { + if (writer->use_small_buffer) { + assert(writer->buffer == NULL); + } + else { assert(writer->buffer != NULL); - assert(PyBytes_CheckExact(writer->buffer)); + if (writer->use_bytearray) + assert(PyByteArray_CheckExact(writer->buffer)); + else + assert(PyBytes_CheckExact(writer->buffer)); assert(Py_REFCNT(writer->buffer) == 1); } - else { - assert(writer->buffer == NULL); + + if (writer->use_bytearray) { + /* bytearray has its own overallocation algorithm, + writer overallocation must be disabled */ + assert(!writer->overallocate); } - start = _PyBytesWriter_AsString(writer); + assert(0 <= writer->allocated); assert(0 <= writer->min_size && writer->min_size <= writer->allocated); /* the last byte must always be null */ + start = _PyBytesWriter_AsString(writer); assert(start[writer->allocated] == 0); end = start + writer->allocated; @@ -3932,8 +3943,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) if (writer->min_size > PY_SSIZE_T_MAX - size) { PyErr_NoMemory(); - _PyBytesWriter_Dealloc(writer); - return NULL; + goto error; } writer->min_size += size; @@ -3950,23 +3960,38 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) pos = _PyBytesWriter_GetPos(writer, str); if (!writer->use_small_buffer) { - /* Note: Don't use a bytearray object because the conversion from - byterray to bytes requires to copy all bytes. */ - if (_PyBytes_Resize(&writer->buffer, allocated)) { - assert(writer->buffer == NULL); - return NULL; + if (writer->use_bytearray) { + if (PyByteArray_Resize(writer->buffer, allocated)) + goto error; + /* writer->allocated can be smaller than writer->buffer->ob_alloc, + but we cannot use ob_alloc because bytes may need to be moved + to use the whole buffer. bytearray uses an internal optimization + to avoid moving or copying bytes when bytes are removed at the + beginning (ex: del bytearray[:1]). */ + } + else { + if (_PyBytes_Resize(&writer->buffer, allocated)) + goto error; } } else { /* convert from stack buffer to bytes object buffer */ assert(writer->buffer == NULL); - writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); + if (writer->use_bytearray) + writer->buffer = PyByteArray_FromStringAndSize(NULL, allocated); + else + writer->buffer = PyBytes_FromStringAndSize(NULL, allocated); if (writer->buffer == NULL) - return NULL; + goto error; if (pos != 0) { - Py_MEMCPY(PyBytes_AS_STRING(writer->buffer), + char *dest; + if (writer->use_bytearray) + dest = PyByteArray_AS_STRING(writer->buffer); + else + dest = PyBytes_AS_STRING(writer->buffer); + Py_MEMCPY(dest, writer->small_buffer, pos); } @@ -3981,6 +4006,10 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) str = _PyBytesWriter_AsString(writer) + pos; _PyBytesWriter_CheckConsistency(writer, str); return str; + +error: + _PyBytesWriter_Dealloc(writer); + return NULL; } /* Allocate the buffer to write size bytes. @@ -4013,7 +4042,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) _PyBytesWriter_CheckConsistency(writer, str); pos = _PyBytesWriter_GetPos(writer, str); - if (pos == 0) { + if (pos == 0 && !writer->use_bytearray) { Py_CLEAR(writer->buffer); /* Get the empty byte string singleton */ result = PyBytes_FromStringAndSize(NULL, 0); @@ -4026,9 +4055,17 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) writer->buffer = NULL; if (pos != writer->allocated) { - if (_PyBytes_Resize(&result, pos)) { - assert(result == NULL); - return NULL; + if (writer->use_bytearray) { + if (PyByteArray_Resize(result, pos)) { + Py_DECREF(result); + return NULL; + } + } + else { + if (_PyBytes_Resize(&result, pos)) { + assert(result == NULL); + return NULL; + } } } } -- cgit v0.12 From 772b2b09f279fdcb01bbd703735d35bd02dd8ec1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 09:56:53 +0200 Subject: Optimize bytearray % args Issue #25399: Don't create temporary bytes objects: modify _PyBytes_Format() to create work directly on bytearray objects. * Rename _PyBytes_Format() to _PyBytes_FormatEx() just in case if something outside CPython uses it * _PyBytes_FormatEx() now uses (char*, Py_ssize_t) for the input string, so bytearray_format() doesn't need tot create a temporary input bytes object * Add use_bytearray parameter to _PyBytes_FormatEx() which is passed to _PyBytesWriter, to create a bytearray buffer instead of a bytes buffer Most formatting operations are now between 2.5 and 5 times faster. --- Include/bytesobject.h | 6 +++++- Objects/bytearrayobject.c | 22 +++++----------------- Objects/bytesobject.c | 41 +++++++++++++++++++++++------------------ 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index fbb6322..b5b37ef 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -62,7 +62,11 @@ PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject *) _PyBytes_Format(PyObject *, PyObject *); +PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( + const char *format, + Py_ssize_t format_len, + PyObject *args, + int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5647b57..e535bce 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -282,26 +282,14 @@ PyByteArray_Concat(PyObject *a, PyObject *b) static PyObject * bytearray_format(PyByteArrayObject *self, PyObject *args) { - PyObject *bytes_in, *bytes_out, *res; - char *bytestring; - - if (self == NULL || !PyByteArray_Check(self) || args == NULL) { + if (self == NULL || !PyByteArray_Check(self)) { PyErr_BadInternalCall(); return NULL; } - bytestring = PyByteArray_AS_STRING(self); - bytes_in = PyBytes_FromString(bytestring); - if (bytes_in == NULL) - return NULL; - bytes_out = _PyBytes_Format(bytes_in, args); - Py_DECREF(bytes_in); - if (bytes_out == NULL) - return NULL; - res = PyByteArray_FromObject(bytes_out); - Py_DECREF(bytes_out); - if (res == NULL) - return NULL; - return res; + + return _PyBytes_FormatEx(PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self), + args, 1); } /* Functions stuffed into the type object */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a1f2958..20b11fb 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -568,28 +568,32 @@ format_obj(PyObject *v, const char **pbuf, Py_ssize_t *plen) /* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * -_PyBytes_Format(PyObject *format, PyObject *args) +_PyBytes_FormatEx(const char *format, Py_ssize_t format_len, + PyObject *args, int use_bytearray) { - char *fmt, *res; + const char *fmt; + char *res; Py_ssize_t arglen, argidx; Py_ssize_t fmtcnt; int args_owned = 0; PyObject *dict = NULL; _PyBytesWriter writer; - if (format == NULL || !PyBytes_Check(format) || args == NULL) { + if (args == NULL) { PyErr_BadInternalCall(); return NULL; } - fmt = PyBytes_AS_STRING(format); - fmtcnt = PyBytes_GET_SIZE(format); + fmt = format; + fmtcnt = format_len; _PyBytesWriter_Init(&writer); + writer.use_bytearray = use_bytearray; res = _PyBytesWriter_Alloc(&writer, fmtcnt); if (res == NULL) return NULL; - writer.overallocate = 1; + if (!use_bytearray) + writer.overallocate = 1; if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); @@ -613,10 +617,8 @@ _PyBytes_Format(PyObject *format, PyObject *args) pos = strchr(fmt + 1, '%'); if (pos != NULL) len = pos - fmt; - else { - len = PyBytes_GET_SIZE(format); - len -= (fmt - PyBytes_AS_STRING(format)); - } + else + len = format_len - (fmt - format); assert(len != 0); Py_MEMCPY(res, fmt, len); @@ -644,7 +646,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) fmt++; if (*fmt == '(') { - char *keystart; + const char *keystart; Py_ssize_t keylen; PyObject *key; int pcount = 1; @@ -924,8 +926,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) "unsupported format character '%c' (0x%x) " "at index %zd", c, c, - (Py_ssize_t)(fmt - 1 - - PyBytes_AsString(format))); + (Py_ssize_t)(fmt - 1 - format)); goto error; } @@ -1028,7 +1029,7 @@ _PyBytes_Format(PyObject *format, PyObject *args) /* If overallocation was disabled, ensure that it was the last write. Otherwise, we missed an optimization */ - assert(writer.overallocate || fmtcnt < 0); + assert(writer.overallocate || fmtcnt < 0 || use_bytearray); } /* until end */ if (argidx < arglen && !dict) { @@ -3233,11 +3234,15 @@ bytes_methods[] = { }; static PyObject * -bytes_mod(PyObject *v, PyObject *w) +bytes_mod(PyObject *self, PyObject *args) { - if (!PyBytes_Check(v)) - Py_RETURN_NOTIMPLEMENTED; - return _PyBytes_Format(v, w); + if (self == NULL || !PyBytes_Check(self)) { + PyErr_BadInternalCall(); + return NULL; + } + + return _PyBytes_FormatEx(PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + args, 0); } static PyNumberMethods bytes_as_number = { -- cgit v0.12 From ebcf9edc05c03af38c01d8aeb05494b68169756c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 10:10:00 +0200 Subject: Document latest optimizations using _PyBytesWriter --- Doc/whatsnew/3.6.rst | 17 +++++++++++++---- Misc/NEWS | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b735e9c..d1e5045 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -141,16 +141,25 @@ Optimizations ============= * The ASCII decoder is now up to 60 times as fast for error handlers: - ``surrogateescape``, ``ignore`` and ``replace``. + ``surrogateescape``, ``ignore`` and ``replace`` (Contributed + by Victor Stinner in :issue:`24870`). * The ASCII and the Latin1 encoders are now up to 3 times as fast for the error - error ``surrogateescape``. + error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). * The UTF-8 encoder is now up to 75 times as fast for error handlers: - ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass``. + ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass`` (Contributed + by Victor Stinner in :issue:`25267`). * The UTF-8 decoder is now up to 15 times as fast for error handlers: - ``ignore``, ``replace`` and ``surrogateescape``. + ``ignore``, ``replace`` and ``surrogateescape`` (Contributed + by Victor Stinner in :issue:`25301`). + +* ``bytes % args`` is now up to 2 times faster. (Contributed by Victor Stinner + in :issue:`25349`). + +* ``bytearray % args`` is now between 2.5 and 5 times faster. (Contributed by + Victor Stinner in :issue:`25399`). Build and C API Changes diff --git a/Misc/NEWS b/Misc/NEWS index 2700ad3..97e5e77 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25399: Optimize bytearray % args using the new private _PyBytesWriter + API. Formatting is now between 2.5 and 5 times faster. + - Issue #25274: sys.setrecursionlimit() now raises a RecursionError if the new recursion limit is too low depending at the current recursion depth. Modify also the "lower-water mark" formula to make it monotonic. This mark is used @@ -19,6 +22,7 @@ Core and Builtins sys.stdout.fileno() fails. - Issue #25349: Optimize bytes % args using the new private _PyBytesWriter API. + Formatting is now up to 2 times faster. - Issue #24806: Prevent builtin types that are not allowed to be subclassed from being subclassed through multiple inheritance. -- cgit v0.12 From 2bf8993db966256d564d87865ceddf0e33c02500 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 11:25:33 +0200 Subject: Optimize bytes.fromhex() and bytearray.fromhex() Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now between 2x and 3.5x faster. Changes: * Use a fast-path working on a char* string for ASCII string * Use a slow-path for non-ASCII string * Replace slow hex_digit_to_int() function with a O(1) lookup in _PyLong_DigitValue precomputed table * Use _PyBytesWriter API to handle the buffer * Add unit tests to check the error position in error messages --- Doc/whatsnew/3.6.rst | 3 ++ Include/bytesobject.h | 3 ++ Include/longobject.h | 2 +- Lib/test/test_bytes.py | 14 +++++ Misc/NEWS | 3 ++ Objects/bytearrayobject.c | 43 +--------------- Objects/bytesobject.c | 128 +++++++++++++++++++++++++++------------------- 7 files changed, 101 insertions(+), 95 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d1e5045..edacea1 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -161,6 +161,9 @@ Optimizations * ``bytearray % args`` is now between 2.5 and 5 times faster. (Contributed by Victor Stinner in :issue:`25399`). +* Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now + between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). + Build and C API Changes ======================= diff --git a/Include/bytesobject.h b/Include/bytesobject.h index b5b37ef..4046c1c 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -67,6 +67,9 @@ PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( Py_ssize_t format_len, PyObject *args, int use_bytearray); +PyAPI_FUNC(PyObject*) _PyBytes_FromHex( + PyObject *string, + int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, diff --git a/Include/longobject.h b/Include/longobject.h index ab92495..9574f05 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -65,7 +65,7 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -/* Used by Python/mystrtoul.c. */ +/* Used by Python/mystrtoul.c and _PyBytes_FromHex(). */ #ifndef Py_LIMITED_API PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; #endif diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 947d959..0fe33b5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -301,6 +301,20 @@ class BaseBytesTest: self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + for data, pos in ( + # invalid first hexadecimal character + ('12 x4 56', 3), + # invalid second hexadecimal character + ('12 3x 56', 4), + # two invalid hexadecimal characters + ('12 xy 56', 3), + # test non-ASCII string + ('12 3\xff 56', 4), + ): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(data) + self.assertIn('at position %s' % pos, str(cm.exception)) + def test_hex(self): self.assertRaises(TypeError, self.type2test.hex) self.assertRaises(TypeError, self.type2test.hex, 1) diff --git a/Misc/NEWS b/Misc/NEWS index 97e5e77..312cde0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25401: Optimize bytes.fromhex() and bytearray.fromhex(): they are now + between 2x and 3.5x faster. + - Issue #25399: Optimize bytearray % args using the new private _PyBytesWriter API. Formatting is now between 2.5 and 5 times faster. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index e535bce..b270fcc 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2823,48 +2823,7 @@ static PyObject * bytearray_fromhex_impl(PyObject*cls, PyObject *string) /*[clinic end generated code: output=df3da60129b3700c input=907bbd2d34d9367a]*/ { - PyObject *newbytes; - char *buf; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - void *data; - unsigned int kind; - - assert(PyUnicode_Check(string)); - if (PyUnicode_READY(string)) - return NULL; - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); - hexlen = PyUnicode_GET_LENGTH(string); - - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); - if (!newbytes) - return NULL; - buf = PyByteArray_AS_STRING(newbytes); - for (i = j = 0; i < hexlen; i += 2) { - /* skip over spaces in the input */ - while (PyUnicode_READ(kind, data, i) == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); - bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); - goto error; - } - buf[j++] = (top << 4) + bot; - } - if (PyByteArray_Resize(newbytes, j) < 0) - goto error; - return newbytes; - - error: - Py_DECREF(newbytes); - return NULL; + return _PyBytes_FromHex(string, 1); } PyDoc_STRVAR(hex__doc__, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 20b11fb..2d4cf4b 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -30,6 +30,10 @@ static PyBytesObject *nullstring; */ #define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +/* Forward declaration */ +Py_LOCAL_INLINE(Py_ssize_t) _PyBytesWriter_GetSize(_PyBytesWriter *writer, + char *str); + /* For PyBytes_FromString(), the parameter `str' points to a null-terminated string containing exactly `size' bytes. @@ -3078,22 +3082,6 @@ bytes_splitlines_impl(PyBytesObject*self, int keepends) ); } -static int -hex_digit_to_int(Py_UCS4 c) -{ - if (c >= 128) - return -1; - if (Py_ISDIGIT(c)) - return c - '0'; - else { - if (Py_ISUPPER(c)) - c = Py_TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - /*[clinic input] @classmethod bytes.fromhex @@ -3111,47 +3099,83 @@ static PyObject * bytes_fromhex_impl(PyTypeObject *type, PyObject *string) /*[clinic end generated code: output=0973acc63661bb2e input=bf4d1c361670acd3]*/ { - PyObject *newstring; + return _PyBytes_FromHex(string, 0); +} + +PyObject* +_PyBytes_FromHex(PyObject *string, int use_bytearray) +{ char *buf; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - void *data; - unsigned int kind; + Py_ssize_t hexlen, invalid_char; + unsigned int top, bot; + Py_UCS1 *str, *end; + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); + writer.use_bytearray = use_bytearray; assert(PyUnicode_Check(string)); if (PyUnicode_READY(string)) return NULL; - kind = PyUnicode_KIND(string); - data = PyUnicode_DATA(string); hexlen = PyUnicode_GET_LENGTH(string); - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newstring = PyBytes_FromStringAndSize(NULL, byteslen); - if (!newstring) + if (!PyUnicode_IS_ASCII(string)) { + void *data = PyUnicode_DATA(string); + unsigned int kind = PyUnicode_KIND(string); + Py_ssize_t i; + + /* search for the first non-ASCII character */ + for (i = 0; i < hexlen; i++) { + if (PyUnicode_READ(kind, data, i) >= 128) + break; + } + invalid_char = i; + goto error; + } + + assert(PyUnicode_KIND(string) == PyUnicode_1BYTE_KIND); + str = PyUnicode_1BYTE_DATA(string); + + /* This overestimates if there are spaces */ + buf = _PyBytesWriter_Alloc(&writer, hexlen / 2); + if (buf == NULL) return NULL; - buf = PyBytes_AS_STRING(newstring); - for (i = j = 0; i < hexlen; i += 2) { + + end = str + hexlen; + while (str < end) { /* skip over spaces in the input */ - while (PyUnicode_READ(kind, data, i) == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(PyUnicode_READ(kind, data, i)); - bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1)); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); + if (*str == ' ') { + do { + str++; + } while (*str == ' '); + if (str >= end) + break; + } + + top = _PyLong_DigitValue[*str]; + if (top >= 16) { + invalid_char = str - PyUnicode_1BYTE_DATA(string); goto error; } - buf[j++] = (top << 4) + bot; + str++; + + bot = _PyLong_DigitValue[*str]; + if (bot >= 16) { + invalid_char = str - PyUnicode_1BYTE_DATA(string); + goto error; + } + str++; + + *buf++ = (unsigned char)((top << 4) + bot); } - if (j != byteslen && _PyBytes_Resize(&newstring, j) < 0) - goto error; - return newstring; + + return _PyBytesWriter_Finish(&writer, buf); error: - Py_XDECREF(newstring); + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", invalid_char); + _PyBytesWriter_Dealloc(&writer); return NULL; } @@ -3888,7 +3912,7 @@ _PyBytesWriter_AsString(_PyBytesWriter *writer) } Py_LOCAL_INLINE(Py_ssize_t) -_PyBytesWriter_GetPos(_PyBytesWriter *writer, char *str) +_PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str) { char *start = _PyBytesWriter_AsString(writer); assert(str != NULL); @@ -3963,7 +3987,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) allocated += allocated / OVERALLOCATE_FACTOR; } - pos = _PyBytesWriter_GetPos(writer, str); + pos = _PyBytesWriter_GetSize(writer, str); if (!writer->use_small_buffer) { if (writer->use_bytearray) { if (PyByteArray_Resize(writer->buffer, allocated)) @@ -4041,33 +4065,33 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) PyObject * _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str) { - Py_ssize_t pos; + Py_ssize_t size; PyObject *result; _PyBytesWriter_CheckConsistency(writer, str); - pos = _PyBytesWriter_GetPos(writer, str); - if (pos == 0 && !writer->use_bytearray) { + size = _PyBytesWriter_GetSize(writer, str); + if (size == 0 && !writer->use_bytearray) { Py_CLEAR(writer->buffer); /* Get the empty byte string singleton */ result = PyBytes_FromStringAndSize(NULL, 0); } else if (writer->use_small_buffer) { - result = PyBytes_FromStringAndSize(writer->small_buffer, pos); + result = PyBytes_FromStringAndSize(writer->small_buffer, size); } else { result = writer->buffer; writer->buffer = NULL; - if (pos != writer->allocated) { + if (size != writer->allocated) { if (writer->use_bytearray) { - if (PyByteArray_Resize(result, pos)) { + if (PyByteArray_Resize(result, size)) { Py_DECREF(result); return NULL; } } else { - if (_PyBytes_Resize(&result, pos)) { + if (_PyBytes_Resize(&result, size)) { assert(result == NULL); return NULL; } -- cgit v0.12 From f091033b149792e4084a479444c39636c7be2cad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 11:59:46 +0200 Subject: Issue #25401: Remove now unused hex_digit_to_int() function --- Objects/bytearrayobject.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b270fcc..2103147 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2789,22 +2789,6 @@ bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) ); } -static int -hex_digit_to_int(Py_UCS4 c) -{ - if (c >= 128) - return -1; - if (Py_ISDIGIT(c)) - return c - '0'; - else { - if (Py_ISUPPER(c)) - c = Py_TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - /*[clinic input] @classmethod bytearray.fromhex -- cgit v0.12 From f6358a7e4c635a74202e73c76a01f5046e1b7c36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 12:02:39 +0200 Subject: _PyBytesWriter_Alloc(): only use 10 bytes of the small buffer in debug mode to enhance code to detect buffer under- and overflow. --- Objects/bytesobject.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2d4cf4b..8810647 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -4053,8 +4053,20 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size) writer->use_small_buffer = 1; #ifdef Py_DEBUG - /* the last byte is reserved, it must be '\0' */ writer->allocated = sizeof(writer->small_buffer) - 1; + /* In debug mode, don't use the full small buffer because it is less + efficient than bytes and bytearray objects to detect buffer underflow + and buffer overflow. Use 10 bytes of the small buffer to test also + code using the smaller buffer in debug mode. + + Don't modify the _PyBytesWriter structure (use a shorter small buffer) + in debug mode to also be able to detect stack overflow when running + tests in debug mode. The _PyBytesWriter is large (more than 512 bytes), + if Py_EnterRecursiveCall() is not used in deep C callback, we may hit a + stack overflow. */ + writer->allocated = Py_MIN(writer->allocated, 10); + /* _PyBytesWriter_CheckConsistency() requires the last byte to be 0, + to detect buffer overflow */ writer->small_buffer[writer->allocated] = 0; #else writer->allocated = sizeof(writer->small_buffer); -- cgit v0.12 From 1285e5c80562ac84ca8bc0ea39b100215d1808a1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 12:10:20 +0200 Subject: Fix compiler warnings (uninitialized variables), false alarms in fact --- Objects/longobject.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 759116a..2cc1585 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1587,7 +1587,7 @@ long_to_decimal_string_internal(PyObject *aa, char **bytes_str) { PyLongObject *scratch, *a; - PyObject *str; + PyObject *str = NULL; Py_ssize_t size, strlen, size_a, i, j; digit *pout, *pin, rem, tenpow; int negative; @@ -1664,7 +1664,6 @@ long_to_decimal_string_internal(PyObject *aa, return -1; } kind = writer->kind; - str = NULL; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, strlen); @@ -1777,7 +1776,7 @@ long_format_binary(PyObject *aa, int base, int alternate, _PyBytesWriter *bytes_writer, char **bytes_str) { PyLongObject *a = (PyLongObject *)aa; - PyObject *v; + PyObject *v = NULL; Py_ssize_t sz; Py_ssize_t size_a; enum PyUnicode_Kind kind; @@ -1834,7 +1833,6 @@ long_format_binary(PyObject *aa, int base, int alternate, if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) return -1; kind = writer->kind; - v = NULL; } else if (bytes_writer) { *bytes_str = _PyBytesWriter_Prepare(bytes_writer, *bytes_str, sz); -- cgit v0.12 From 2ec8063cc960d32e244dc6a27567f66a447bbda3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:32:13 +0200 Subject: Modify _PyBytes_DecodeEscapeRecode() to use _PyBytesAPI * Don't overallocate by 400% when recode is needed: only overallocate on demand using _PyBytesWriter. * Use _PyLong_DigitValue to convert hexadecimal digit to int * Create _PyBytes_DecodeEscapeRecode() subfunction --- Include/longobject.h | 3 +- Objects/bytesobject.c | 131 ++++++++++++++++++++++++++++---------------------- 2 files changed, 75 insertions(+), 59 deletions(-) diff --git a/Include/longobject.h b/Include/longobject.h index 9574f05..eaf7a7e 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -65,7 +65,8 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -/* Used by Python/mystrtoul.c and _PyBytes_FromHex(). */ +/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), + _PyBytes_DecodeEscapeRecode(), etc. */ #ifndef Py_LIMITED_API PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; #endif diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8810647..556b480 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1068,6 +1068,42 @@ bytes_dealloc(PyObject *op) the string is UTF-8 encoded and should be re-encoded in the specified encoding. */ +static char * +_PyBytes_DecodeEscapeRecode(const char **s, const char *end, + const char *errors, const char *recode_encoding, + _PyBytesWriter *writer, char *p) +{ + PyObject *u, *w; + const char* t; + + t = *s; + /* Decode non-ASCII bytes as UTF-8. */ + while (t < end && (*t & 0x80)) + t++; + u = PyUnicode_DecodeUTF8(*s, t - *s, errors); + if (u == NULL) + return NULL; + + /* Recode them in target encoding. */ + w = PyUnicode_AsEncodedString(u, recode_encoding, errors); + Py_DECREF(u); + if (w == NULL) + return NULL; + assert(PyBytes_Check(w)); + + /* Append bytes to output buffer. */ + writer->min_size--; /* substract 1 preallocated byte */ + p = _PyBytesWriter_WriteBytes(writer, p, + PyBytes_AS_STRING(w), + PyBytes_GET_SIZE(w)); + Py_DECREF(w); + if (p == NULL) + return NULL; + + *s = t; + return p; +} + PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, @@ -1075,54 +1111,42 @@ PyObject *PyBytes_DecodeEscape(const char *s, const char *recode_encoding) { int c; - char *p, *buf; + char *p; const char *end; - PyObject *v; - Py_ssize_t newlen = recode_encoding ? 4*len:len; - v = PyBytes_FromStringAndSize((char *)NULL, newlen); - if (v == NULL) + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); + + p = _PyBytesWriter_Alloc(&writer, len); + if (p == NULL) return NULL; - p = buf = PyBytes_AsString(v); + writer.overallocate = 1; + end = s + len; while (s < end) { if (*s != '\\') { non_esc: - if (recode_encoding && (*s & 0x80)) { - PyObject *u, *w; - char *r; - const char* t; - Py_ssize_t rn; - t = s; - /* Decode non-ASCII bytes as UTF-8. */ - while (t < end && (*t & 0x80)) t++; - u = PyUnicode_DecodeUTF8(s, t - s, errors); - if(!u) goto failed; - - /* Recode them in target encoding. */ - w = PyUnicode_AsEncodedString( - u, recode_encoding, errors); - Py_DECREF(u); - if (!w) goto failed; - - /* Append bytes to output buffer. */ - assert(PyBytes_Check(w)); - r = PyBytes_AS_STRING(w); - rn = PyBytes_GET_SIZE(w); - Py_MEMCPY(p, r, rn); - p += rn; - Py_DECREF(w); - s = t; - } else { + if (!(recode_encoding && (*s & 0x80))) { *p++ = *s++; } + else { + /* non-ASCII character and need to recode */ + p = _PyBytes_DecodeEscapeRecode(&s, end, + errors, recode_encoding, + &writer, p); + if (p == NULL) + goto failed; + } continue; } + s++; - if (s==end) { + if (s == end) { PyErr_SetString(PyExc_ValueError, "Trailing \\ in string"); goto failed; } + switch (*s++) { /* XXX This assumes ASCII! */ case '\n': break; @@ -1147,28 +1171,18 @@ PyObject *PyBytes_DecodeEscape(const char *s, *p++ = c; break; case 'x': - if (s+1 < end && Py_ISXDIGIT(s[0]) && Py_ISXDIGIT(s[1])) { - unsigned int x = 0; - c = Py_CHARMASK(*s); - s++; - if (Py_ISDIGIT(c)) - x = c - '0'; - else if (Py_ISLOWER(c)) - x = 10 + c - 'a'; - else - x = 10 + c - 'A'; - x = x << 4; - c = Py_CHARMASK(*s); - s++; - if (Py_ISDIGIT(c)) - x += c - '0'; - else if (Py_ISLOWER(c)) - x += 10 + c - 'a'; - else - x += 10 + c - 'A'; - *p++ = x; - break; + if (s+1 < end) { + int digit1, digit2; + digit1 = _PyLong_DigitValue[Py_CHARMASK(s[0])]; + digit2 = _PyLong_DigitValue[Py_CHARMASK(s[1])]; + if (digit1 < 16 && digit2 < 16) { + *p++ = (unsigned char)((digit1 << 4) + digit2); + s += 2; + break; + } } + /* invalid hexadecimal digits */ + if (!errors || strcmp(errors, "strict") == 0) { PyErr_Format(PyExc_ValueError, "invalid \\x escape at position %d", @@ -1190,6 +1204,7 @@ PyObject *PyBytes_DecodeEscape(const char *s, if (s < end && Py_ISXDIGIT(s[0])) s++; /* and a hexdigit */ break; + default: *p++ = '\\'; s--; @@ -1197,11 +1212,11 @@ PyObject *PyBytes_DecodeEscape(const char *s, UTF-8 bytes may follow. */ } } - if (p-buf < newlen) - _PyBytes_Resize(&v, p - buf); - return v; + + return _PyBytesWriter_Finish(&writer, p); + failed: - Py_DECREF(v); + _PyBytesWriter_Dealloc(&writer); return NULL; } -- cgit v0.12 From f2eafa323bc16e11dd4564ec68f27e8ea8b9c254 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:44:29 +0200 Subject: Split PyBytes_FromObject() into subfunctions --- Objects/bytesobject.c | 185 +++++++++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 71 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 556b480..1aae9e3 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3384,88 +3384,99 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return PyBytes_FromObject(x); } -PyObject * -PyBytes_FromObject(PyObject *x) +static PyObject* +_PyBytes_FromBuffer(PyObject *x) { - PyObject *new, *it; - Py_ssize_t i, size; + PyObject *new; + Py_buffer view; - if (x == NULL) { - PyErr_BadInternalCall(); + if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) return NULL; - } - if (PyBytes_CheckExact(x)) { - Py_INCREF(x); - return x; - } + new = PyBytes_FromStringAndSize(NULL, view.len); + if (!new) + goto fail; + if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, + &view, view.len, 'C') < 0) + goto fail; + PyBuffer_Release(&view); + return new; - /* Use the modern buffer interface */ - if (PyObject_CheckBuffer(x)) { - Py_buffer view; - if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) - return NULL; - new = PyBytes_FromStringAndSize(NULL, view.len); - if (!new) - goto fail; - if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, - &view, view.len, 'C') < 0) - goto fail; - PyBuffer_Release(&view); - return new; - fail: - Py_XDECREF(new); - PyBuffer_Release(&view); - return NULL; - } - if (PyUnicode_Check(x)) { - PyErr_SetString(PyExc_TypeError, - "cannot convert unicode object to bytes"); +fail: + Py_XDECREF(new); + PyBuffer_Release(&view); + return NULL; +} + +static PyObject* +_PyBytes_FromList(PyObject *x) +{ + PyObject *new; + Py_ssize_t i; + Py_ssize_t value; + char *str; + + new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); + if (new == NULL) return NULL; - } + str = ((PyBytesObject *)new)->ob_sval; - if (PyList_CheckExact(x)) { - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - for (i = 0; i < Py_SIZE(x); i++) { - Py_ssize_t value = PyNumber_AsSsize_t( - PyList_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) { - Py_DECREF(new); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - Py_DECREF(new); - return NULL; - } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + for (i = 0; i < Py_SIZE(x); i++) { + value = PyNumber_AsSsize_t(PyList_GET_ITEM(x, i), PyExc_ValueError); + if (value == -1 && PyErr_Occurred()) + goto error; + + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; } - return new; + *str++ = (char) value; } - if (PyTuple_CheckExact(x)) { - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - for (i = 0; i < Py_SIZE(x); i++) { - Py_ssize_t value = PyNumber_AsSsize_t( - PyTuple_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) { - Py_DECREF(new); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - Py_DECREF(new); - return NULL; - } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + return new; + +error: + Py_DECREF(new); + return NULL; +} + +static PyObject* +_PyBytes_FromTuple(PyObject *x) +{ + PyObject *new; + Py_ssize_t i; + Py_ssize_t value; + char *str; + + new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); + if (new == NULL) + return NULL; + str = ((PyBytesObject *)new)->ob_sval; + + for (i = 0; i < Py_SIZE(x); i++) { + value = PyNumber_AsSsize_t(PyTuple_GET_ITEM(x, i), PyExc_ValueError); + if (value == -1 && PyErr_Occurred()) + goto error; + + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; } - return new; + *str++ = (char) value; } + return new; + +error: + Py_DECREF(new); + return NULL; +} + +static PyObject * +_PyBytes_FromIterator(PyObject *x) +{ + PyObject *new, *it; + Py_ssize_t i, size; /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); @@ -3533,6 +3544,38 @@ PyBytes_FromObject(PyObject *x) return NULL; } +PyObject * +PyBytes_FromObject(PyObject *x) +{ + if (x == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + if (PyBytes_CheckExact(x)) { + Py_INCREF(x); + return x; + } + + /* Use the modern buffer interface */ + if (PyObject_CheckBuffer(x)) + return _PyBytes_FromBuffer(x); + + if (PyList_CheckExact(x)) + return _PyBytes_FromList(x); + + if (PyTuple_CheckExact(x)) + return _PyBytes_FromTuple(x); + + if (PyUnicode_Check(x)) { + PyErr_SetString(PyExc_TypeError, + "cannot convert unicode object to bytes"); + return NULL; + } + + return _PyBytes_FromIterator(x); +} + static PyObject * str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { -- cgit v0.12 From 3c50ce39bfe0b5b905432b195c5bb1cf939f4273 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:50:40 +0200 Subject: Factorize _PyBytes_FromList() and _PyBytes_FromTuple() code using a C macro --- Objects/bytesobject.c | 89 ++++++++++++++++++++------------------------------- 1 file changed, 35 insertions(+), 54 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 1aae9e3..e54f299 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3408,68 +3408,49 @@ fail: return NULL; } +#define _PyBytes_FROM_LIST_BODY(x, GET_ITEM) \ + do { \ + PyObject *bytes; \ + Py_ssize_t i; \ + Py_ssize_t value; \ + char *str; \ + PyObject *item; \ + \ + bytes = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); \ + if (bytes == NULL) \ + return NULL; \ + str = ((PyBytesObject *)bytes)->ob_sval; \ + \ + for (i = 0; i < Py_SIZE(x); i++) { \ + item = GET_ITEM((x), i); \ + value = PyNumber_AsSsize_t(item, PyExc_ValueError); \ + if (value == -1 && PyErr_Occurred()) \ + goto error; \ + \ + if (value < 0 || value >= 256) { \ + PyErr_SetString(PyExc_ValueError, \ + "bytes must be in range(0, 256)"); \ + goto error; \ + } \ + *str++ = (char) value; \ + } \ + return bytes; \ + \ + error: \ + Py_DECREF(bytes); \ + return NULL; \ + } while (0) + static PyObject* _PyBytes_FromList(PyObject *x) { - PyObject *new; - Py_ssize_t i; - Py_ssize_t value; - char *str; - - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - str = ((PyBytesObject *)new)->ob_sval; - - for (i = 0; i < Py_SIZE(x); i++) { - value = PyNumber_AsSsize_t(PyList_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) - goto error; - - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - *str++ = (char) value; - } - return new; - -error: - Py_DECREF(new); - return NULL; + _PyBytes_FROM_LIST_BODY(x, PyList_GET_ITEM); } static PyObject* _PyBytes_FromTuple(PyObject *x) { - PyObject *new; - Py_ssize_t i; - Py_ssize_t value; - char *str; - - new = PyBytes_FromStringAndSize(NULL, Py_SIZE(x)); - if (new == NULL) - return NULL; - str = ((PyBytesObject *)new)->ob_sval; - - for (i = 0; i < Py_SIZE(x); i++) { - value = PyNumber_AsSsize_t(PyTuple_GET_ITEM(x, i), PyExc_ValueError); - if (value == -1 && PyErr_Occurred()) - goto error; - - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - *str++ = (char) value; - } - return new; - -error: - Py_DECREF(new); - return NULL; + _PyBytes_FROM_LIST_BODY(x, PyTuple_GET_ITEM); } static PyObject * -- cgit v0.12 From c5c3ba4becf98b37b4dd92333c144b0d95faef03 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 13:56:47 +0200 Subject: Add _PyBytesWriter_Resize() function This function gives a control to the buffer size without using min_size. --- Include/bytesobject.h | 19 ++++++++++++++++++- Objects/bytesobject.c | 48 ++++++++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 4046c1c..8469112 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -178,7 +178,9 @@ PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size); -/* Add *size* bytes to the buffer. +/* Ensure that the buffer is large enough to write *size* bytes. + Add size to the writer minimum size (min_size attribute). + str is the current pointer inside the buffer. Return the updated current pointer inside the buffer. Raise an exception and return NULL on error. */ @@ -186,6 +188,21 @@ PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size); +/* Resize the buffer to make it larger. + The new buffer may be larger than size bytes because of overallocation. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. + + Note: size must be greater than the number of allocated bytes in the writer. + + This function doesn't use the writer minimum size (min_size attribute). + + See also _PyBytesWriter_Prepare(). + */ +PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + /* Write bytes. Raise an exception and return NULL on error. */ PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e54f299..ae7b1ea 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3997,29 +3997,14 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str) } void* -_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) +_PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size) { Py_ssize_t allocated, pos; _PyBytesWriter_CheckConsistency(writer, str); - assert(size >= 0); - - if (size == 0) { - /* nothing to do */ - return str; - } - - if (writer->min_size > PY_SSIZE_T_MAX - size) { - PyErr_NoMemory(); - goto error; - } - writer->min_size += size; - - allocated = writer->allocated; - if (writer->min_size <= allocated) - return str; + assert(writer->allocated < size); - allocated = writer->min_size; + allocated = size; if (writer->overallocate && allocated <= (PY_SSIZE_T_MAX - allocated / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ @@ -4080,6 +4065,33 @@ error: return NULL; } +void* +_PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size) +{ + Py_ssize_t new_min_size; + + _PyBytesWriter_CheckConsistency(writer, str); + assert(size >= 0); + + if (size == 0) { + /* nothing to do */ + return str; + } + + if (writer->min_size > PY_SSIZE_T_MAX - size) { + PyErr_NoMemory(); + _PyBytesWriter_Dealloc(writer); + return NULL; + } + new_min_size = writer->min_size + size; + + if (new_min_size > writer->allocated) + str = _PyBytesWriter_Resize(writer, str, new_min_size); + + writer->min_size = new_min_size; + return str; +} + /* Allocate the buffer to write size bytes. Return the pointer to the beginning of buffer data. Raise an exception and return NULL on error. */ -- cgit v0.12 From c3d2bc19e47892e19c9ad6a048047964dbb78cb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 14:15:49 +0200 Subject: Use _PyBytesWriter in _PyBytes_FromIterator() --- Objects/bytesobject.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index ae7b1ea..c10dbdf 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3456,23 +3456,23 @@ _PyBytes_FromTuple(PyObject *x) static PyObject * _PyBytes_FromIterator(PyObject *x) { - PyObject *new, *it; + char *str; + PyObject *it; Py_ssize_t i, size; + _PyBytesWriter writer; + + _PyBytesWriter_Init(&writer); /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; - /* Allocate an extra byte to prevent PyBytes_FromStringAndSize() from - returning a shared empty bytes string. This required because we - want to call _PyBytes_Resize() the returned object, which we can - only do on bytes objects with refcount == 1. */ - if (size == 0) - size = 1; - new = PyBytes_FromStringAndSize(NULL, size); - if (new == NULL) + + str = _PyBytesWriter_Alloc(&writer, size); + if (str == NULL) return NULL; - assert(Py_REFCNT(new) == 1); + writer.overallocate = 1; + size = writer.allocated; /* Get the iterator */ it = PyObject_GetIter(x); @@ -3507,21 +3507,20 @@ _PyBytes_FromIterator(PyObject *x) /* Append the byte */ if (i >= size) { - size = 2 * size + 1; - if (_PyBytes_Resize(&new, size) < 0) - goto error; + str = _PyBytesWriter_Resize(&writer, str, size+1); + if (str == NULL) + return NULL; + size = writer.allocated; } - ((PyBytesObject *)new)->ob_sval[i] = (char) value; + *str++ = (char) value; } - _PyBytes_Resize(&new, i); - - /* Clean up and return success */ Py_DECREF(it); - return new; + + return _PyBytesWriter_Finish(&writer, str); error: + _PyBytesWriter_Dealloc(&writer); Py_XDECREF(it); - Py_XDECREF(new); return NULL; } -- cgit v0.12 From 1bfe9305859cf280464ed1f40fc9e2793b0f03b7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:02:35 +0200 Subject: Issue #25384: Fix binascii.rledecode_hqx() Fix usage of _PyBytesWriter API. Use the new _PyBytesWriter_Resize() function instead of _PyBytesWriter_Prepare(). --- Lib/test/test_binascii.py | 16 +++++++++++++++- Modules/binascii.c | 11 +++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 0ab46bc..fbc933e4 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -159,11 +159,25 @@ class BinASCIITest(unittest.TestCase): # Then calculate the hexbin4 binary-to-ASCII translation rle = binascii.rlecode_hqx(self.data) a = binascii.b2a_hqx(self.type2test(rle)) + b, _ = binascii.a2b_hqx(self.type2test(a)) res = binascii.rledecode_hqx(b) - self.assertEqual(res, self.rawdata) + def test_rle(self): + # test repetition with a repetition longer than the limit of 255 + data = (b'a' * 100 + b'b' + b'c' * 300) + + encoded = binascii.rlecode_hqx(data) + self.assertEqual(encoded, + (b'a\x90d' # 'a' * 100 + b'b' # 'b' + b'c\x90\xff' # 'c' * 255 + b'c\x90-')) # 'c' * 45 + + decoded = binascii.rledecode_hqx(encoded) + self.assertEqual(decoded, data) + def test_hex(self): # test hexlification s = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' diff --git a/Modules/binascii.c b/Modules/binascii.c index dfa5535..a1070b7 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -800,14 +800,15 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len * 2; + out_len = in_len; out_data = _PyBytesWriter_Alloc(&writer, out_len); if (out_data == NULL) return NULL; /* Use overallocation */ writer.overallocate = 1; - out_len_left = writer.allocated; + out_len = writer.allocated; + out_len_left = out_len; /* ** We need two macros here to get/put bytes and handle @@ -830,10 +831,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) overallocate the buffer anymore */ \ writer.overallocate = 0; \ } \ - out_data = _PyBytesWriter_Prepare(&writer, out_data, 1); \ + out_data = _PyBytesWriter_Resize(&writer, out_data, \ + writer.allocated + 1); \ if (out_data == NULL) \ goto error; \ - out_len_left = writer.allocated; \ + out_len_left = writer.allocated - out_len - 1; \ + out_len = writer.allocated; \ } \ *out_data++ = b; \ } while(0) -- cgit v0.12 From f9c9a3fedfcf9da646cd0183b93bc00459cff164 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:20:07 +0200 Subject: Refactor binascii.rledecode_hqx() Rewrite the code to handle the output buffer. --- Modules/binascii.c | 53 +++++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/Modules/binascii.c b/Modules/binascii.c index a1070b7..ccd81fa 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -784,7 +784,7 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) { unsigned char *in_data, *out_data; unsigned char in_byte, in_repeat; - Py_ssize_t in_len, out_len, out_len_left; + Py_ssize_t in_len; _PyBytesWriter writer; in_data = data->buf; @@ -800,15 +800,12 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) return PyErr_NoMemory(); /* Allocate a buffer of reasonable size. Resized when needed */ - out_len = in_len; - out_data = _PyBytesWriter_Alloc(&writer, out_len); + out_data = _PyBytesWriter_Alloc(&writer, in_len); if (out_data == NULL) return NULL; /* Use overallocation */ writer.overallocate = 1; - out_len = writer.allocated; - out_len_left = out_len; /* ** We need two macros here to get/put bytes and handle @@ -823,24 +820,6 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) b = *in_data++; \ } while(0) -#define OUTBYTE(b) \ - do { \ - if ( --out_len_left < 0 ) { \ - if (in_len <= 0) { \ - /* We are done after this write, no need to \ - overallocate the buffer anymore */ \ - writer.overallocate = 0; \ - } \ - out_data = _PyBytesWriter_Resize(&writer, out_data, \ - writer.allocated + 1); \ - if (out_data == NULL) \ - goto error; \ - out_len_left = writer.allocated - out_len - 1; \ - out_len = writer.allocated; \ - } \ - *out_data++ = b; \ - } while(0) - /* ** Handle first byte separately (since we have to get angry ** in case of an orphaned RLE code). @@ -849,6 +828,10 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) if (in_byte == RUNCHAR) { INBYTE(in_repeat); + /* only 1 byte will be written, but 2 bytes were preallocated: + substract 1 byte to prevent overallocation */ + writer.min_size--; + if (in_repeat != 0) { /* Note Error, not Incomplete (which is at the end ** of the string only). This is a programmer error. @@ -856,9 +839,9 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) PyErr_SetString(Error, "Orphaned RLE code at start"); goto error; } - OUTBYTE(RUNCHAR); + *out_data++ = RUNCHAR; } else { - OUTBYTE(in_byte); + *out_data++ = in_byte; } while( in_len > 0 ) { @@ -866,18 +849,32 @@ binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) if (in_byte == RUNCHAR) { INBYTE(in_repeat); + /* only 1 byte will be written, but 2 bytes were preallocated: + substract 1 byte to prevent overallocation */ + writer.min_size--; + if ( in_repeat == 0 ) { /* Just an escaped RUNCHAR value */ - OUTBYTE(RUNCHAR); + *out_data++ = RUNCHAR; } else { /* Pick up value and output a sequence of it */ in_byte = out_data[-1]; + + /* enlarge the buffer if needed */ + if (in_repeat > 1) { + /* -1 because we already preallocated 1 byte */ + out_data = _PyBytesWriter_Prepare(&writer, out_data, + in_repeat - 1); + if (out_data == NULL) + goto error; + } + while ( --in_repeat > 0 ) - OUTBYTE(in_byte); + *out_data++ = in_byte; } } else { /* Normal byte */ - OUTBYTE(in_byte); + *out_data++ = in_byte; } } return _PyBytesWriter_Finish(&writer, out_data); -- cgit v0.12 From 83ff8a68324dfe54af8baf7e67f81b2db567ae7a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 15:28:59 +0200 Subject: test_bytes: new try to fix test on '%p' formatter on Windows --- Lib/test/test_bytes.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 0fe33b5..87799df 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -839,12 +839,22 @@ class BytesTest(BaseBytesTest, unittest.TestCase): b'i=-123') self.assertEqual(PyBytes_FromFormat(b'x=%x', c_int(0xabc)), b'x=abc') + + sizeof_ptr = ctypes.sizeof(c_char_p) + + if os.name == 'nt': + # Windows (MSCRT) + ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + def ptr_formatter(ptr): + return (ptr_format % ptr) + else: + # UNIX (glibc) + def ptr_formatter(ptr): + return '%#x' % ptr + ptr = 0xabcdef - expected = [b'ptr=%#x' % ptr] - win_format = 'ptr=0x%0{}X'.format(2 * ctypes.sizeof(c_char_p)) - expected.append((win_format % ptr).encode('ascii')) - self.assertIn(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), - expected) + self.assertEqual(PyBytes_FromFormat(b'ptr=%p', c_char_p(ptr)), + ('ptr=' + ptr_formatter(ptr)).encode('ascii')) self.assertEqual(PyBytes_FromFormat(b's=%s', c_char_p(b'cstr')), b's=cstr') @@ -859,7 +869,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MIN, str), (b'%zd', c_ssize_t, _testcapi.PY_SSIZE_T_MAX, str), (b'%zu', c_size_t, size_max, str), - (b'%p', c_char_p, size_max, lambda value: '%#x' % value), + (b'%p', c_char_p, size_max, ptr_formatter), ): self.assertEqual(PyBytes_FromFormat(formatstr, ctypes_type(value)), py_formatter(value).encode('ascii')), -- cgit v0.12 From 91108f049ffb35b4b9f0353a41d9c6097b37046d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Oct 2015 18:25:31 +0200 Subject: Issue #25210: Change error message of do_richcompare() Don't add parenthesis to type names. Add also quotes around the type names. Before: TypeError: unorderable types: int() < NoneType() After: TypeError: '<' not supported between instances of 'int' and 'NoneType' --- Doc/howto/argparse.rst | 3 ++- Doc/library/enum.rst | 2 +- Doc/library/pathlib.rst | 2 +- Objects/object.c | 5 ++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 510d1d4..9c111b4 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -547,7 +547,8 @@ And this is what it gives: Traceback (most recent call last): File "prog.py", line 11, in if args.verbosity >= 2: - TypeError: unorderable types: NoneType() >= int() + TypeError: '>=' not supported between instances of 'NoneType' and 'int' + * First output went well, and fixes the bug we had before. That is, we want any value >= 2 to be as verbose as possible. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 18519f0..0fbbf5a 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -257,7 +257,7 @@ members are not integers (but see `IntEnum`_ below):: >>> Color.red < Color.blue Traceback (most recent call last): File "", line 1, in - TypeError: unorderable types: Color() < Color() + TypeError: '<' not supported between instances of 'Color' and 'Color' Equality comparisons are defined though:: diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2f06544..ff5196d 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -195,7 +195,7 @@ Paths of a different flavour compare unequal and cannot be ordered:: >>> PureWindowsPath('foo') < PurePosixPath('foo') Traceback (most recent call last): File "", line 1, in - TypeError: unorderable types: PureWindowsPath() < PurePosixPath() + TypeError: '<' not supported between instances of 'PureWindowsPath' and 'PurePosixPath' Operators diff --git a/Objects/object.c b/Objects/object.c index 6fc4df1..e1718ea 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -686,11 +686,10 @@ do_richcompare(PyObject *v, PyObject *w, int op) res = (v != w) ? Py_True : Py_False; break; default: - /* XXX Special-case None so it doesn't show as NoneType() */ PyErr_Format(PyExc_TypeError, - "unorderable types: %.100s() %s %.100s()", - v->ob_type->tp_name, + "'%s' not supported between instances of '%.100s' and '%.100s'", opstrings[op], + v->ob_type->tp_name, w->ob_type->tp_name); return NULL; } -- cgit v0.12 From 1286d14500e0cf30e900ecf0b597ee1c308e0dfe Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 14 Oct 2015 23:16:57 -0700 Subject: Improve variable names and constant expressions --- Modules/_collectionsmodule.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index cef92c0..06f85e9 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -303,8 +303,8 @@ deque_append(dequeobject *deque, PyObject *item) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *rv = deque_popleft(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_popleft(deque, NULL); + Py_DECREF(olditem); } Py_RETURN_NONE; } @@ -331,8 +331,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { - PyObject *rv = deque_pop(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_pop(deque, NULL); + Py_DECREF(olditem); } Py_RETURN_NONE; } @@ -423,8 +423,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *rv = deque_popleft(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_popleft(deque, NULL); + Py_DECREF(olditem); } } return finalize_iterator(it); @@ -487,8 +487,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, maxlen)) { - PyObject *rv = deque_pop(deque, NULL); - Py_DECREF(rv); + PyObject *olditem = deque_pop(deque, NULL); + Py_DECREF(olditem); } } return finalize_iterator(it); @@ -1283,6 +1283,7 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) PyObject *item; Py_ssize_t index; Py_ssize_t indexlo = deque->leftindex; + Py_ssize_t indexhigh; for (b = deque->leftblock; b != deque->rightblock; b = b->rightlink) { for (index = indexlo; index < BLOCKLEN ; index++) { @@ -1291,7 +1292,8 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) } indexlo = 0; } - for (index = indexlo; index <= deque->rightindex; index++) { + indexhigh = deque->rightindex; + for (index = indexlo; index <= indexhigh; index++) { item = b->data[index]; Py_VISIT(item); } -- cgit v0.12 From bc00341105afed2255ecd139ca6ca381a482b1f9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 14 Oct 2015 23:33:23 -0700 Subject: Use unsigned division --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 06f85e9..bc225a5 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1482,7 +1482,7 @@ deque_sizeof(dequeobject *deque, void *unused) Py_ssize_t blocks; res = sizeof(dequeobject); - blocks = (deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN; + blocks = (size_t)(deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN; assert(deque->leftindex + Py_SIZE(deque) - 1 == (blocks - 1) * BLOCKLEN + deque->rightindex); res += blocks * sizeof(block); -- cgit v0.12 From a4b13d00200053e5985c089af45be34cdbffd1eb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 15 Oct 2015 08:05:31 -0700 Subject: Rewrap comment. --- Modules/_collectionsmodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index bc225a5..85a0793 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -272,10 +272,10 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); /* The deque's size limit is d.maxlen. The limit can be zero or positive. * If there is no limit, then d.maxlen == -1. * - * After an item is added to a deque, we check to see if the size has grown past - * the limit. If it has, we get the size back down to the limit by popping an - * item off of the opposite end. The methods that can trigger this are append(), - * appendleft(), extend(), and extendleft(). + * After an item is added to a deque, we check to see if the size has + * grown past the limit. If it has, we get the size back down to the limit + * by popping an item off of the opposite end. The methods that can + * trigger this are append(), appendleft(), extend(), and extendleft(). * * The macro to check whether a deque needs to be trimmed uses a single * unsigned test that returns true whenever 0 <= maxlen < Py_SIZE(deque). -- cgit v0.12 From 1eca237c2f3d5c150cf9184fa7c9d9b137d1187c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 15 Oct 2015 23:25:53 -0700 Subject: Remove old Todo entry that isn't going to happen. --- Modules/_collectionsmodule.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 85a0793..1acbf86 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -156,12 +156,6 @@ freeblock(block *b) } } -/* XXX Todo: - If aligned memory allocations become available, make the - deque object 64 byte aligned so that all of the fields - can be retrieved or updated in a single cache line. -*/ - static PyObject * deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { -- cgit v0.12 From c0d91aff9a3b91307b26e8b7c34dfbf27bbdd43a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 16 Oct 2015 12:21:37 -0700 Subject: Upgrade the imp module's deprecation to DeprecationWarning. --- Doc/whatsnew/3.6.rst | 3 +++ Lib/imp.py | 2 +- Lib/pkgutil.py | 2 +- Misc/NEWS | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index edacea1..b374280 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -228,6 +228,9 @@ Changes in the Python API now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. +* The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of + :exc:`PendingDeprecationWarning`. + Changes in the C API -------------------- diff --git a/Lib/imp.py b/Lib/imp.py index f6fff44..b339952 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -30,7 +30,7 @@ import warnings warnings.warn("the imp module is deprecated in favour of importlib; " "see the module's documentation for alternative uses", - PendingDeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=2) # DEPRECATED SEARCH_ERROR = 0 diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index fc4a074..203d515 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -180,7 +180,7 @@ iter_importer_modules.register( def _import_imp(): global imp with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) imp = importlib.import_module('imp') class ImpImporter: diff --git a/Misc/NEWS b/Misc/NEWS index bcc85a7..6e3e2c3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -63,6 +63,8 @@ Core and Builtins Library ------- +- Move the imp module from a PendingDeprecationWarning to DeprecationWarning. + - Issue #25407: Remove mentions of the formatter module being removed in Python 3.6. -- cgit v0.12 From 9b63868f77c6782bba6b3ceed8e2058b471b6fd5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 16 Oct 2015 15:14:27 -0700 Subject: Issue #25154: Deprecate the pyvenv script. This was done so as to move users to `python3 -m venv` which prevents confusion over which Python interpreter will be used in the virtual environment when more than one is installed. --- Doc/whatsnew/3.6.rst | 5 ++++- Misc/NEWS | 6 ++++++ Tools/scripts/pyvenv | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b374280..22513cf8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -197,7 +197,10 @@ Deprecated functions and types of the C API Deprecated features ------------------- -* None yet. +* The ``pyvenv`` script has been deprecated in favour of ``python3 -m venv``. + This prevents confusion as to what Python interpreter ``pyvenv`` is + connected to and thus what Python interpreter will be used by the virtual + environment. See :issue:`25154`. Removed diff --git a/Misc/NEWS b/Misc/NEWS index 6e3e2c3..7d2c79f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -289,6 +289,12 @@ Windows - Issue #25022: Removed very outdated PC/example_nt/ directory. +Tools/Demos +----------- + +- Issue #25154: The pyvenv script has been deprecated in favour of + `python3 -m venv`. + What's New in Python 3.5.1 release candidate 1? =============================================== diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 978d691..1043efc 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -1,6 +1,12 @@ #!/usr/bin/env python3 if __name__ == '__main__': import sys + import pathlib + + executable = pathlib.Path(sys.executable or 'python3').name + print('WARNING: the pyenv script is deprecated in favour of ' + '`{} -m venv`'.format(exeutable, file=sys.stderr)) + rc = 1 try: import venv -- cgit v0.12 From 67317742162dd5b9728672b2ff7ed21e2aa7d2fa Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 16 Oct 2015 20:45:53 -0400 Subject: Issue 25422: Add tests for multi-line string tokenization. Also remove truncated tokens. --- Lib/test/test_tokenize.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 3b17ca6..b74396f 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -24,8 +24,7 @@ class TokenizeTest(TestCase): if type == ENDMARKER: break type = tok_name[type] - result.append(" %(type)-10.10s %(token)-13.13r %(start)s %(end)s" % - locals()) + result.append(f" {type:10} {token!r:13} {start} {end}") self.assertEqual(result, [" ENCODING 'utf-8' (0, 0) (0, 0)"] + expected.rstrip().splitlines()) @@ -132,18 +131,18 @@ def k(x): self.check_tokenize("x = 0xfffffffffff", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '0xffffffffff (1, 4) (1, 17) + NUMBER '0xfffffffffff' (1, 4) (1, 17) """) self.check_tokenize("x = 123141242151251616110", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) - NUMBER '123141242151 (1, 4) (1, 25) + NUMBER '123141242151251616110' (1, 4) (1, 25) """) self.check_tokenize("x = -15921590215012591", """\ NAME 'x' (1, 0) (1, 1) OP '=' (1, 2) (1, 3) OP '-' (1, 4) (1, 5) - NUMBER '159215902150 (1, 5) (1, 22) + NUMBER '15921590215012591' (1, 5) (1, 22) """) def test_float(self): @@ -307,6 +306,33 @@ def k(x): OP '+' (1, 28) (1, 29) STRING 'RB"abc"' (1, 30) (1, 37) """) + # Check 0, 1, and 2 character string prefixes. + self.check_tokenize(r'"a\ +de\ +fg"', """\ + STRING '"a\\\\\\nde\\\\\\nfg"\' (1, 0) (3, 3) + """) + self.check_tokenize(r'u"a\ +de"', """\ + STRING 'u"a\\\\\\nde"\' (1, 0) (2, 3) + """) + self.check_tokenize(r'rb"a\ +d"', """\ + STRING 'rb"a\\\\\\nd"\' (1, 0) (2, 2) + """) + self.check_tokenize(r'"""a\ +b"""', """\ + STRING '\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'u"""a\ +b"""', """\ + STRING 'u\"\""a\\\\\\nb\"\""' (1, 0) (2, 4) + """) + self.check_tokenize(r'rb"""a\ +b\ +c"""', """\ + STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) + """) def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ @@ -505,7 +531,7 @@ def k(x): # Methods self.check_tokenize("@staticmethod\ndef foo(x,y): pass", """\ OP '@' (1, 0) (1, 1) - NAME 'staticmethod (1, 1) (1, 13) + NAME 'staticmethod' (1, 1) (1, 13) NEWLINE '\\n' (1, 13) (1, 14) NAME 'def' (2, 0) (2, 3) NAME 'foo' (2, 4) (2, 7) -- cgit v0.12 From 20151f50f6942f32584ece6e751e1edb59699007 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 16 Oct 2015 22:47:29 -0700 Subject: Issue #25414: Remove unnecessary tests that can never succeed. --- Modules/_collectionsmodule.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1acbf86..ceba037 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -110,12 +110,6 @@ static PyTypeObject deque_type; #define CHECK_NOT_END(link) #endif -/* To prevent len from overflowing PY_SSIZE_T_MAX, we refuse to - allocate new blocks if the current len is nearing overflow. -*/ - -#define MAX_DEQUE_LEN (PY_SSIZE_T_MAX - 3*BLOCKLEN) - /* A simple freelisting scheme is used to minimize calls to the memory allocator. It accommodates common use cases where new blocks are being added at about the same rate as old blocks are being freed. @@ -128,11 +122,6 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(Py_ssize_t len) { block *b; - if (len >= MAX_DEQUE_LEN) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more blocks to the deque"); - return NULL; - } if (numfreeblocks) { numfreeblocks--; return freeblocks[numfreeblocks]; @@ -676,9 +665,6 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) if (deque->maxlen >= 0 && n > deque->maxlen) n = deque->maxlen; - if (n > MAX_DEQUE_LEN) - return PyErr_NoMemory(); - deque->state++; for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { @@ -709,7 +695,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) return (PyObject *)deque; } - if ((size_t)size > MAX_DEQUE_LEN / (size_t)n) { + if ((size_t)size > PY_SSIZE_T_MAX / (size_t)n) { return PyErr_NoMemory(); } -- cgit v0.12 From 6dc9ce19230e102ff6a64d998304292799949418 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Tue, 20 Oct 2015 01:07:53 -0400 Subject: Remove double 'error'. --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 22513cf8..b500550 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -144,7 +144,7 @@ Optimizations ``surrogateescape``, ``ignore`` and ``replace`` (Contributed by Victor Stinner in :issue:`24870`). -* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error +* The ASCII and the Latin1 encoders are now up to 3 times as fast for the error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). * The UTF-8 encoder is now up to 75 times as fast for error handlers: -- cgit v0.12 From 0f43bb160e65b4acbee898896b4fd4cc1420807f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 20 Oct 2015 00:03:33 -0700 Subject: Only update the state variable once per iteration. --- Modules/_collectionsmodule.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ceba037..aa879be 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -269,7 +269,6 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); static PyObject * deque_append(dequeobject *deque, PyObject *item) { - deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) @@ -288,6 +287,8 @@ deque_append(dequeobject *deque, PyObject *item) if (NEEDS_TRIM(deque, deque->maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } Py_RETURN_NONE; } @@ -297,7 +298,6 @@ PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) { - deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) @@ -316,6 +316,8 @@ deque_appendleft(dequeobject *deque, PyObject *item) if (NEEDS_TRIM(deque, deque->maxlen)) { PyObject *olditem = deque_pop(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } Py_RETURN_NONE; } @@ -387,7 +389,6 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - deque->state++; if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) { @@ -408,6 +409,8 @@ deque_extend(dequeobject *deque, PyObject *iterable) if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } } return finalize_iterator(it); @@ -451,7 +454,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - deque->state++; if (deque->leftindex == 0) { block *b = newblock(Py_SIZE(deque)); if (b == NULL) { @@ -472,6 +474,8 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_pop(deque, NULL); Py_DECREF(olditem); + } else { + deque->state++; } } return finalize_iterator(it); -- cgit v0.12 From 6a4d52e86521cfabd0646daacd31ba08e6098a78 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Thu, 22 Oct 2015 07:49:36 +0300 Subject: Issue #25210: Add some basic tests for the new exception message --- Lib/test/test_richcmp.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 1582caa..58729a9 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -253,6 +253,31 @@ class MiscTest(unittest.TestCase): self.assertTrue(a != b) self.assertTrue(a < b) + def test_exception_message(self): + class Spam: + pass + + tests = [ + (lambda: 42 < None, r"'<' .* of 'int' and 'NoneType'"), + (lambda: None < 42, r"'<' .* of 'NoneType' and 'int'"), + (lambda: 42 > None, r"'>' .* of 'int' and 'NoneType'"), + (lambda: "foo" < None, r"'<' .* of 'str' and 'NoneType'"), + (lambda: "foo" >= 666, r"'>=' .* of 'str' and 'int'"), + (lambda: 42 <= None, r"'<=' .* of 'int' and 'NoneType'"), + (lambda: 42 >= None, r"'>=' .* of 'int' and 'NoneType'"), + (lambda: 42 < [], r"'<' .* of 'int' and 'list'"), + (lambda: () > [], r"'>' .* of 'tuple' and 'list'"), + (lambda: None >= None, r"'>=' .* of 'NoneType' and 'NoneType'"), + (lambda: Spam() < 42, r"'<' .* of 'Spam' and 'int'"), + (lambda: 42 < Spam(), r"'<' .* of 'int' and 'Spam'"), + (lambda: Spam() <= Spam(), r"'<=' .* of 'Spam' and 'Spam'"), + ] + for i, test in enumerate(tests): + with self.subTest(test=i): + with self.assertRaisesRegex(TypeError, test[1]): + test[0]() + + class DictTest(unittest.TestCase): def test_dicts(self): -- cgit v0.12 From db41fd43cf60ed22e3c3b2f55101b48f9ae3d215 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 22 Oct 2015 22:48:16 -0700 Subject: Removed unused parameter --- Modules/_collectionsmodule.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index aa879be..ba8238c 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -120,7 +120,7 @@ static Py_ssize_t numfreeblocks = 0; static block *freeblocks[MAXFREEBLOCKS]; static block * -newblock(Py_ssize_t len) { +newblock(void) { block *b; if (numfreeblocks) { numfreeblocks--; @@ -156,7 +156,7 @@ deque_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (deque == NULL) return NULL; - b = newblock(0); + b = newblock(); if (b == NULL) { Py_DECREF(deque); return NULL; @@ -270,7 +270,7 @@ static PyObject * deque_append(dequeobject *deque, PyObject *item) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) return NULL; b->leftlink = deque->rightblock; @@ -299,7 +299,7 @@ static PyObject * deque_appendleft(dequeobject *deque, PyObject *item) { if (deque->leftindex == 0) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) return NULL; b->rightlink = deque->leftblock; @@ -390,7 +390,7 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) { Py_DECREF(item); Py_DECREF(it); @@ -455,7 +455,7 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque->leftindex == 0) { - block *b = newblock(Py_SIZE(deque)); + block *b = newblock(); if (b == NULL) { Py_DECREF(item); Py_DECREF(it); @@ -586,7 +586,7 @@ deque_clear(dequeobject *deque) adversary could cause it to never terminate). */ - b = newblock(0); + b = newblock(); if (b == NULL) { PyErr_Clear(); goto alternate_method; @@ -672,7 +672,7 @@ deque_inplace_repeat(dequeobject *deque, Py_ssize_t n) deque->state++; for (i = 0 ; i < n-1 ; ) { if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(Py_SIZE(deque) + i); + block *b = newblock(); if (b == NULL) { Py_SIZE(deque) += i; return NULL; @@ -786,7 +786,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) while (n > 0) { if (leftindex == 0) { if (b == NULL) { - b = newblock(len); + b = newblock(); if (b == NULL) goto done; } @@ -830,7 +830,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) while (n < 0) { if (rightindex == BLOCKLEN - 1) { if (b == NULL) { - b = newblock(len); + b = newblock(); if (b == NULL) goto done; } -- cgit v0.12 From 5b6f3644e67d6bc597ff246a1e75902d36d5699e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 23 Oct 2015 13:24:03 -0700 Subject: Issue #24633: Removes automatic rename of site-packages/README since README.txt is now committed. --- Tools/msi/lib/lib_files.wxs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index fa79a8d..804ab01 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -64,16 +64,10 @@ - - - - - - -- cgit v0.12 From f1c47e47514e1cf1006dc21de386af7ee25f8db5 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 25 Oct 2015 17:40:31 -0700 Subject: Issue #25154: Make the file argument apply to the print function and not str.format call. --- Tools/scripts/pyvenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 1043efc..1eed3ff 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -5,7 +5,7 @@ if __name__ == '__main__': executable = pathlib.Path(sys.executable or 'python3').name print('WARNING: the pyenv script is deprecated in favour of ' - '`{} -m venv`'.format(exeutable, file=sys.stderr)) + '`{} -m venv`'.format(exeutable), file=sys.stderr) rc = 1 try: -- cgit v0.12 From 1c8222c80a8a534bd9357aafc3fba7b6927efc15 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Mon, 26 Oct 2015 04:37:55 -0400 Subject: Issue 25311: Add support for f-strings to tokenize.py. Also added some comments to explain what's happening, since it's not so obvious. --- Lib/test/test_tokenize.py | 17 +++++++ Lib/tokenize.py | 118 ++++++++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index b74396f..90438e7 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -333,6 +333,23 @@ b\ c"""', """\ STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4) """) + self.check_tokenize('f"abc"', """\ + STRING 'f"abc"' (1, 0) (1, 6) + """) + self.check_tokenize('fR"a{b}c"', """\ + STRING 'fR"a{b}c"' (1, 0) (1, 9) + """) + self.check_tokenize('f"""abc"""', """\ + STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10) + """) + self.check_tokenize(r'f"abc\ +def"', """\ + STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4) + """) + self.check_tokenize(r'Rf"abc\ +def"', """\ + STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4) + """) def test_function(self): self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\ diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 65d06e5..2237c3a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -29,6 +29,7 @@ from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper from itertools import chain +import itertools as _itertools import re import sys from token import * @@ -131,7 +132,28 @@ Floatnumber = group(Pointfloat, Expfloat) Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') Number = group(Imagnumber, Floatnumber, Intnumber) -StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?' +# Return the empty string, plus all of the valid string prefixes. +def _all_string_prefixes(): + # The valid string prefixes. Only contain the lower case versions, + # and don't contain any permuations (include 'fr', but not + # 'rf'). The various permutations will be generated. + _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + # if we add binary f-strings, add: ['fb', 'fbr'] + result = set(['']) + for prefix in _valid_string_prefixes: + for t in _itertools.permutations(prefix): + # create a list with upper and lower versions of each + # character + for u in _itertools.product(*[(c, c.upper()) for c in t]): + result.add(''.join(u)) + return result + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +# Note that since _all_string_prefixes includes the empty string, +# StringPrefix can be the empty string (making it optional). +StringPrefix = group(*_all_string_prefixes()) # Tail end of ' string. Single = r"[^'\\]*(?:\\.[^'\\]*)*'" @@ -169,50 +191,25 @@ ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" + PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -def _compile(expr): - return re.compile(expr, re.UNICODE) - -endpats = {"'": Single, '"': Double, - "'''": Single3, '"""': Double3, - "r'''": Single3, 'r"""': Double3, - "b'''": Single3, 'b"""': Double3, - "R'''": Single3, 'R"""': Double3, - "B'''": Single3, 'B"""': Double3, - "br'''": Single3, 'br"""': Double3, - "bR'''": Single3, 'bR"""': Double3, - "Br'''": Single3, 'Br"""': Double3, - "BR'''": Single3, 'BR"""': Double3, - "rb'''": Single3, 'rb"""': Double3, - "Rb'''": Single3, 'Rb"""': Double3, - "rB'''": Single3, 'rB"""': Double3, - "RB'''": Single3, 'RB"""': Double3, - "u'''": Single3, 'u"""': Double3, - "U'''": Single3, 'U"""': Double3, - 'r': None, 'R': None, 'b': None, 'B': None, - 'u': None, 'U': None} - -triple_quoted = {} -for t in ("'''", '"""', - "r'''", 'r"""', "R'''", 'R"""', - "b'''", 'b"""', "B'''", 'B"""', - "br'''", 'br"""', "Br'''", 'Br"""', - "bR'''", 'bR"""', "BR'''", 'BR"""', - "rb'''", 'rb"""', "rB'''", 'rB"""', - "Rb'''", 'Rb"""', "RB'''", 'RB"""', - "u'''", 'u"""', "U'''", 'U"""', - ): - triple_quoted[t] = t -single_quoted = {} -for t in ("'", '"', - "r'", 'r"', "R'", 'R"', - "b'", 'b"', "B'", 'B"', - "br'", 'br"', "Br'", 'Br"', - "bR'", 'bR"', "BR'", 'BR"' , - "rb'", 'rb"', "rB'", 'rB"', - "Rb'", 'Rb"', "RB'", 'RB"' , - "u'", 'u"', "U'", 'U"', - ): - single_quoted[t] = t +# For a given string prefix plus quotes, endpats maps it to a regex +# to match the remainder of that string. _prefix can be empty, for +# a normal single or triple quoted string (with no prefix). +endpats = {} +for _prefix in _all_string_prefixes(): + endpats[_prefix + "'"] = Single + endpats[_prefix + '"'] = Double + endpats[_prefix + "'''"] = Single3 + endpats[_prefix + '"""'] = Double3 + +# A set of all of the single and triple quoted string prefixes, +# including the opening quotes. +single_quoted = set() +triple_quoted = set() +for t in _all_string_prefixes(): + for u in (t + '"', t + "'"): + single_quoted.add(u) + for u in (t + '"""', t + "'''"): + triple_quoted.add(u) tabsize = 8 @@ -626,6 +623,7 @@ def _tokenize(readline, encoding): yield stashed stashed = None yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: endprog = _compile(endpats[token]) endmatch = endprog.match(line, pos) @@ -638,19 +636,37 @@ def _tokenize(readline, encoding): contstr = line[start:] contline = line break - elif initial in single_quoted or \ - token[:2] in single_quoted or \ - token[:3] in single_quoted: + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come afer + # triple quote checking (above). + elif (initial in single_quoted or + token[:2] in single_quoted or + token[:3] in single_quoted): if token[-1] == '\n': # continued string strstart = (lnum, start) - endprog = _compile(endpats[initial] or - endpats[token[1]] or - endpats[token[2]]) + # Again, using the first 3 chars of the + # token. This is looking for the matching end + # regex for the correct type of quote + # character. So it's really looking for + # endpats["'"] or endpats['"'], by trying to + # skip string prefix characters, if any. + endprog = _compile(endpats.get(initial) or + endpats.get(token[1]) or + endpats.get(token[2])) contstr, needcont = line[start:], 1 contline = line break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name if token in ('async', 'await'): if async_def: -- cgit v0.12 From dd1e670758794b1dccdd42aa1f8ccddabdd406c2 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 26 Oct 2015 17:11:04 -0700 Subject: Fix a variable typo by switching to a f-string. --- Tools/scripts/pyvenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv index 1eed3ff..1fb42c6 100755 --- a/Tools/scripts/pyvenv +++ b/Tools/scripts/pyvenv @@ -5,7 +5,7 @@ if __name__ == '__main__': executable = pathlib.Path(sys.executable or 'python3').name print('WARNING: the pyenv script is deprecated in favour of ' - '`{} -m venv`'.format(exeutable), file=sys.stderr) + f'`{executable} -m venv`', file=sys.stderr) rc = 1 try: -- cgit v0.12 From 213b4056a64a670587419c8a15a724faeb789221 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 30 Oct 2015 14:41:06 -0700 Subject: Issue #25487: Fix tests not updated when the imp module moved to a DeprecationWarning. Thanks to Martin Panter for finding the tests. --- Lib/modulefinder.py | 2 +- Lib/test/test_imp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 50f2462..4a2f1b5 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -10,7 +10,7 @@ import types import struct import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp # XXX Clean up once str8's cstor matches bytes. diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index ee9ee1a..efb0384 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -12,7 +12,7 @@ from test import support import unittest import warnings with warnings.catch_warnings(): - warnings.simplefilter('ignore', PendingDeprecationWarning) + warnings.simplefilter('ignore', DeprecationWarning) import imp -- cgit v0.12 From 9785261d68f62ce33968d15c7a0f10f1f8209807 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 1 Nov 2015 17:14:27 +0200 Subject: Issue #18973: Command-line interface of the calendar module now uses argparse instead of optparse. --- Lib/calendar.py | 81 ++++++++++++++++++++++++++--------------------- Lib/test/test_calendar.py | 8 ++--- Misc/NEWS | 3 ++ 3 files changed, 52 insertions(+), 40 deletions(-) diff --git a/Lib/calendar.py b/Lib/calendar.py index 02050ea..4ff154c 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -605,51 +605,63 @@ def timegm(tuple): def main(args): - import optparse - parser = optparse.OptionParser(usage="usage: %prog [options] [year [month]]") - parser.add_option( + import argparse + parser = argparse.ArgumentParser() + textgroup = parser.add_argument_group('text only arguments') + htmlgroup = parser.add_argument_group('html only arguments') + textgroup.add_argument( "-w", "--width", - dest="width", type="int", default=2, - help="width of date column (default 2, text only)" + type=int, default=2, + help="width of date column (default 2)" ) - parser.add_option( + textgroup.add_argument( "-l", "--lines", - dest="lines", type="int", default=1, - help="number of lines for each week (default 1, text only)" + type=int, default=1, + help="number of lines for each week (default 1)" ) - parser.add_option( + textgroup.add_argument( "-s", "--spacing", - dest="spacing", type="int", default=6, - help="spacing between months (default 6, text only)" + type=int, default=6, + help="spacing between months (default 6)" ) - parser.add_option( + textgroup.add_argument( "-m", "--months", - dest="months", type="int", default=3, - help="months per row (default 3, text only)" + type=int, default=3, + help="months per row (default 3)" ) - parser.add_option( + htmlgroup.add_argument( "-c", "--css", - dest="css", default="calendar.css", - help="CSS to use for page (html only)" + default="calendar.css", + help="CSS to use for page" ) - parser.add_option( + parser.add_argument( "-L", "--locale", - dest="locale", default=None, + default=None, help="locale to be used from month and weekday names" ) - parser.add_option( + parser.add_argument( "-e", "--encoding", - dest="encoding", default=None, - help="Encoding to use for output." + default=None, + help="encoding to use for output" ) - parser.add_option( + parser.add_argument( "-t", "--type", - dest="type", default="text", + default="text", choices=("text", "html"), help="output type (text or html)" ) + parser.add_argument( + "year", + nargs='?', type=int, + help="year number (1-9999)" + ) + parser.add_argument( + "month", + nargs='?', type=int, + help="month number (1-12, text only)" + ) - (options, args) = parser.parse_args(args) + options = parser.parse_args(args[1:]) if options.locale and not options.encoding: parser.error("if --locale is specified --encoding is required") @@ -667,10 +679,10 @@ def main(args): encoding = sys.getdefaultencoding() optdict = dict(encoding=encoding, css=options.css) write = sys.stdout.buffer.write - if len(args) == 1: + if options.year is None: write(cal.formatyearpage(datetime.date.today().year, **optdict)) - elif len(args) == 2: - write(cal.formatyearpage(int(args[1]), **optdict)) + elif options.month is None: + write(cal.formatyearpage(options.year, **optdict)) else: parser.error("incorrect number of arguments") sys.exit(1) @@ -680,18 +692,15 @@ def main(args): else: cal = TextCalendar() optdict = dict(w=options.width, l=options.lines) - if len(args) != 3: + if options.month is None: optdict["c"] = options.spacing optdict["m"] = options.months - if len(args) == 1: + if options.year is None: result = cal.formatyear(datetime.date.today().year, **optdict) - elif len(args) == 2: - result = cal.formatyear(int(args[1]), **optdict) - elif len(args) == 3: - result = cal.formatmonth(int(args[1]), int(args[2]), **optdict) + elif options.month is None: + result = cal.formatyear(options.year, **optdict) else: - parser.error("incorrect number of arguments") - sys.exit(1) + result = cal.formatmonth(options.year, options.month, **optdict) write = sys.stdout.write if options.encoding: result = result.encode(options.encoding) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index 80ed632..d9d3128 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -702,19 +702,19 @@ class CommandLineTestCase(unittest.TestCase): def assertFailure(self, *args): rc, stdout, stderr = assert_python_failure('-m', 'calendar', *args) - self.assertIn(b'Usage:', stderr) + self.assertIn(b'usage:', stderr) self.assertEqual(rc, 2) def test_help(self): stdout = self.run_ok('-h') - self.assertIn(b'Usage:', stdout) + self.assertIn(b'usage:', stdout) self.assertIn(b'calendar.py', stdout) self.assertIn(b'--help', stdout) def test_illegal_arguments(self): self.assertFailure('-z') - #self.assertFailure('spam') - #self.assertFailure('2004', 'spam') + self.assertFailure('spam') + self.assertFailure('2004', 'spam') self.assertFailure('-t', 'html', '2004', '1') def test_output_current_year(self): diff --git a/Misc/NEWS b/Misc/NEWS index b67a3a5..5722a3a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -66,6 +66,9 @@ Core and Builtins Library ------- +- Issue #18973: Command-line interface of the calendar module now uses argparse + instead of optparse. + - Issue #25510: fileinput.FileInput.readline() now returns b'' instead of '' at the end if the FileInput was opened with binary mode. Patch by Ryosuke Ito. -- cgit v0.12 From 67b97b8f8dc792aa7e1130a52a0ccc5069c400ed Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 1 Nov 2015 23:57:37 -0500 Subject: Move the initial start-search out of the main loop so it can be factored-out later. --- Modules/_collectionsmodule.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ba8238c..c12d43e 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1017,11 +1017,12 @@ deque_len(dequeobject *deque) static PyObject * deque_index(dequeobject *deque, PyObject *args) { - Py_ssize_t i, start=0, stop=Py_SIZE(deque); + Py_ssize_t i, n, start=0, stop=Py_SIZE(deque); PyObject *v, *item; block *b = deque->leftblock; Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; + int cmp; if (!PyArg_ParseTuple(args, "O|O&O&:index", &v, _PyEval_SliceIndex, &start, @@ -1039,22 +1040,32 @@ deque_index(dequeobject *deque, PyObject *args) } if (stop > Py_SIZE(deque)) stop = Py_SIZE(deque); + if (start > stop) + start = stop; + assert(0 <= start && start <= stop && stop <= Py_SIZE(deque)); - for (i=0 ; i= start) { - int cmp; - CHECK_NOT_END(b); - item = b->data[index]; - cmp = PyObject_RichCompareBool(item, v, Py_EQ); - if (cmp > 0) - return PyLong_FromSsize_t(i); - else if (cmp < 0) - return NULL; - if (start_state != deque->state) { - PyErr_SetString(PyExc_RuntimeError, - "deque mutated during iteration"); - return NULL; - } + /* XXX Replace this loop with faster code from deque_item() */ + for (i=0 ; irightlink; + index = 0; + } + } + + n = stop - i; + while (n--) { + CHECK_NOT_END(b); + item = b->data[index]; + cmp = PyObject_RichCompareBool(item, v, Py_EQ); + if (cmp > 0) + return PyLong_FromSsize_t(stop - (n + 1)); + else if (cmp < 0) + return NULL; + if (start_state != deque->state) { + PyErr_SetString(PyExc_RuntimeError, + "deque mutated during iteration"); + return NULL; } index++; if (index == BLOCKLEN) { -- cgit v0.12 From a1fc949b5ab8911a803eee691e6eea55cec43eeb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Nov 2015 00:39:56 -0500 Subject: Issue #24379: Revert the operator.subscript patch (dccc4e63aef5) pending resolution of the related refcnt leak. --- Doc/library/operator.rst | 15 --------------- Doc/whatsnew/3.6.rst | 8 -------- Lib/operator.py | 28 +--------------------------- Lib/test/test_operator.py | 33 --------------------------------- 4 files changed, 1 insertion(+), 83 deletions(-) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 0695391..c01e63b 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -333,21 +333,6 @@ expect a function argument. [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] -.. data:: subscript - - A helper to turn subscript notation into indexing objects. This can be - used to create item access patterns ahead of time to pass them into - various subscriptable objects. - - For example: - - * ``subscript[5] == 5`` - * ``subscript[3:7:2] == slice(3, 7, 2)`` - * ``subscript[5, 8] == (5, 8)`` - - .. versionadded:: 3.6 - - .. function:: methodcaller(name[, args...]) Return a callable object that calls the method *name* on its operand. If diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b500550..0d387bb 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -104,14 +104,6 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) -operator --------- - -New object :data:`operator.subscript` makes it easier to create complex -indexers. For example: ``subscript[0:10:2] == slice(0, 10, 2)`` -(Contributed by Joe Jevnik in :issue:`24379`.) - - pickle ------ diff --git a/Lib/operator.py b/Lib/operator.py index bc2a947..0e2e53e 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -17,7 +17,7 @@ __all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'subscript', 'truediv', 'truth', 'xor'] + 'setitem', 'sub', 'truediv', 'truth', 'xor'] from builtins import abs as _abs @@ -408,32 +408,6 @@ def ixor(a, b): return a -@object.__new__ # create a singleton instance -class subscript: - """ - A helper to turn subscript notation into indexing objects. This can be - used to create item access patterns ahead of time to pass them into - various subscriptable objects. - - For example: - subscript[5] == 5 - subscript[3:7:2] == slice(3, 7, 2) - subscript[5, 8] == (5, 8) - """ - __slots__ = () - - def __new__(cls): - raise TypeError("cannot create '{}' instances".format(cls.__name__)) - - @staticmethod - def __getitem__(key): - return key - - @staticmethod - def __reduce__(): - return 'subscript' - - try: from _operator import * except ImportError: diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 27501c2..54fd1f4 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -596,38 +596,5 @@ class CCOperatorPickleTestCase(OperatorPickleTestCase, unittest.TestCase): module2 = c_operator -class SubscriptTestCase: - def test_subscript(self): - subscript = self.module.subscript - self.assertIsNone(subscript[None]) - self.assertEqual(subscript[0], 0) - self.assertEqual(subscript[0:1:2], slice(0, 1, 2)) - self.assertEqual( - subscript[0, ..., :2, ...], - (0, Ellipsis, slice(2), Ellipsis), - ) - - def test_pickle(self): - from operator import subscript - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - self.assertIs( - pickle.loads(pickle.dumps(subscript, proto)), - subscript, - ) - - def test_singleton(self): - with self.assertRaises(TypeError): - type(self.module.subscript)() - - def test_immutable(self): - with self.assertRaises(AttributeError): - self.module.subscript.attr = None - - -class PySubscriptTestCase(SubscriptTestCase, PyOperatorTestCase): - pass - - if __name__ == "__main__": unittest.main() -- cgit v0.12 From df8f5b56c9dc5e950f39ed0221212256a39c1835 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 2 Nov 2015 07:27:40 -0500 Subject: Minor cleanup. --- Modules/_collectionsmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index c9e4568..1ca6c72 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1060,7 +1060,7 @@ deque_index(dequeobject *deque, PyObject *args) cmp = PyObject_RichCompareBool(item, v, Py_EQ); if (cmp > 0) return PyLong_FromSsize_t(stop - (n + 1)); - else if (cmp < 0) + if (cmp < 0) return NULL; if (start_state != deque->state) { PyErr_SetString(PyExc_RuntimeError, -- cgit v0.12 From 2753a096e0365bf93e1fd3a3e20735f598c14e91 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Nov 2015 14:34:51 +0100 Subject: locale.delocalize(): only call localeconv() once --- Lib/locale.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/locale.py b/Lib/locale.py index ceaa6d8..074f6e0 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -303,12 +303,16 @@ def str(val): def delocalize(string): "Parses a string as a normalized number according to the locale settings." + + conv = localeconv() + #First, get rid of the grouping - ts = localeconv()['thousands_sep'] + ts = conv['thousands_sep'] if ts: string = string.replace(ts, '') + #next, replace the decimal point with a dot - dd = localeconv()['decimal_point'] + dd = conv['decimal_point'] if dd: string = string.replace(dd, '.') return string -- cgit v0.12 From a78c7954d549fba967a842143b4e3050ff488468 Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 12:45:05 -0500 Subject: Issue 25483: Add an opcode to make f-string formatting more robust. --- Include/ceval.h | 8 ++ Include/opcode.h | 1 + Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 2 + Python/ceval.c | 57 ++++++++++ Python/compile.c | 98 ++++++----------- Python/importlib_external.h | 206 +++++++++++++++++------------------ Python/opcode_targets.h | 2 +- 8 files changed, 207 insertions(+), 170 deletions(-) diff --git a/Include/ceval.h b/Include/ceval.h index b5373a9..d194044 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -206,6 +206,14 @@ PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void); #endif +/* Masks and values used by FORMAT_VALUE opcode. */ +#define FVC_MASK 0x3 +#define FVC_NONE 0x0 +#define FVC_STR 0x1 +#define FVC_REPR 0x2 +#define FVC_ASCII 0x3 +#define FVS_MASK 0x4 +#define FVS_HAVE_SPEC 0x4 #ifdef __cplusplus } diff --git a/Include/opcode.h b/Include/opcode.h index 3f917fb..b265368 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -122,6 +122,7 @@ extern "C" { #define BUILD_TUPLE_UNPACK 152 #define BUILD_SET_UNPACK 153 #define SETUP_ASYNC_WITH 154 +#define FORMAT_VALUE 155 /* EXCEPT_HANDLER is a special, implicit block type which is created when entering an except handler. It is not an opcode but we define it here diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index e0a1200..c58a62a 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,12 +223,13 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3350).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index 4c826a7..71ce672 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -214,4 +214,6 @@ def_op('BUILD_MAP_UNPACK_WITH_CALL', 151) def_op('BUILD_TUPLE_UNPACK', 152) def_op('BUILD_SET_UNPACK', 153) +def_op('FORMAT_VALUE', 155) + del def_op, name_op, jrel_op, jabs_op diff --git a/Python/ceval.c b/Python/ceval.c index f9e8ef8..60db703 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3363,6 +3363,63 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) DISPATCH(); } + TARGET(FORMAT_VALUE) { + /* Handles f-string value formatting. */ + PyObject *result; + PyObject *fmt_spec; + PyObject *value; + PyObject *(*conv_fn)(PyObject *); + int which_conversion = oparg & FVC_MASK; + int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; + + fmt_spec = have_fmt_spec ? POP() : NULL; + value = TOP(); + + /* See if any conversion is specified. */ + switch (which_conversion) { + case FVC_STR: conv_fn = PyObject_Str; break; + case FVC_REPR: conv_fn = PyObject_Repr; break; + case FVC_ASCII: conv_fn = PyObject_ASCII; break; + + /* Must be 0 (meaning no conversion), since only four + values are allowed by (oparg & FVC_MASK). */ + default: conv_fn = NULL; break; + } + + /* If there's a conversion function, call it and replace + value with that result. Otherwise, just use value, + without conversion. */ + if (conv_fn) { + result = conv_fn(value); + Py_DECREF(value); + if (!result) { + Py_XDECREF(fmt_spec); + goto error; + } + value = result; + } + + /* If value is a unicode object, and there's no fmt_spec, + then we know the result of format(value) is value + itself. In that case, skip calling format(). I plan to + move this optimization in to PyObject_Format() + itself. */ + if (PyUnicode_CheckExact(value) && fmt_spec == NULL) { + /* Do nothing, just transfer ownership to result. */ + result = value; + } else { + /* Actually call format(). */ + result = PyObject_Format(value, fmt_spec); + Py_DECREF(value); + Py_XDECREF(fmt_spec); + if (!result) + goto error; + } + + SET_TOP(result); + DISPATCH(); + } + TARGET(EXTENDED_ARG) { opcode = NEXTOP(); oparg = oparg<<16 | NEXTARG(); diff --git a/Python/compile.c b/Python/compile.c index 5b53de3..030afed 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1067,6 +1067,10 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg) return 1; case GET_YIELD_FROM_ITER: return 0; + case FORMAT_VALUE: + /* If there's a fmt_spec on the stack, we go from 2->1, + else 1->1. */ + return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0; default: return PY_INVALID_STACK_EFFECT; } @@ -3241,83 +3245,47 @@ compiler_joined_str(struct compiler *c, expr_ty e) return 1; } -/* Note that this code uses the builtin functions format(), str(), - repr(), and ascii(). You can break this code, or make it do odd - things, by redefining those functions. */ +/* Used to implement f-strings. Format a single value. */ static int compiler_formatted_value(struct compiler *c, expr_ty e) { - PyObject *conversion_name = NULL; - - static PyObject *format_string; - static PyObject *str_string; - static PyObject *repr_string; - static PyObject *ascii_string; - - if (!format_string) { - format_string = PyUnicode_InternFromString("format"); - if (!format_string) - return 0; - } - - if (!str_string) { - str_string = PyUnicode_InternFromString("str"); - if (!str_string) - return 0; - } - - if (!repr_string) { - repr_string = PyUnicode_InternFromString("repr"); - if (!repr_string) - return 0; - } - if (!ascii_string) { - ascii_string = PyUnicode_InternFromString("ascii"); - if (!ascii_string) - return 0; - } + /* Our oparg encodes 2 pieces of information: the conversion + character, and whether or not a format_spec was provided. + + Convert the conversion char to 2 bits: + None: 000 0x0 FVC_NONE + !s : 001 0x1 FVC_STR + !r : 010 0x2 FVC_REPR + !a : 011 0x3 FVC_ASCII + + next bit is whether or not we have a format spec: + yes : 100 0x4 + no : 000 0x0 + */ - ADDOP_NAME(c, LOAD_GLOBAL, format_string, names); + int oparg; - /* If needed, convert via str, repr, or ascii. */ - if (e->v.FormattedValue.conversion != -1) { - switch (e->v.FormattedValue.conversion) { - case 's': - conversion_name = str_string; - break; - case 'r': - conversion_name = repr_string; - break; - case 'a': - conversion_name = ascii_string; - break; - default: - PyErr_SetString(PyExc_SystemError, - "Unrecognized conversion character"); - return 0; - } - ADDOP_NAME(c, LOAD_GLOBAL, conversion_name, names); - } - - /* Evaluate the value. */ + /* Evaluate the expression to be formatted. */ VISIT(c, expr, e->v.FormattedValue.value); - /* If needed, convert via str, repr, or ascii. */ - if (conversion_name) { - /* Call the function we previously pushed. */ - ADDOP_I(c, CALL_FUNCTION, 1); + switch (e->v.FormattedValue.conversion) { + case 's': oparg = FVC_STR; break; + case 'r': oparg = FVC_REPR; break; + case 'a': oparg = FVC_ASCII; break; + case -1: oparg = FVC_NONE; break; + default: + PyErr_SetString(PyExc_SystemError, + "Unrecognized conversion character"); + return 0; } - - /* If we have a format spec, use format(value, format_spec). Otherwise, - use the single argument form. */ if (e->v.FormattedValue.format_spec) { + /* Evaluate the format spec, and update our opcode arg. */ VISIT(c, expr, e->v.FormattedValue.format_spec); - ADDOP_I(c, CALL_FUNCTION, 2); - } else { - /* No format spec specified, call format(value). */ - ADDOP_I(c, CALL_FUNCTION, 1); + oparg |= FVS_HAVE_SPEC; } + /* And push our opcode and oparg */ + ADDOP_I(c, FORMAT_VALUE, oparg); return 1; } diff --git a/Python/importlib_external.h b/Python/importlib_external.h index c4b6dc9..dd322a9 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -258,7 +258,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, - 1,17,1,13,1,5,1,114,55,0,0,0,105,22,13,0, + 1,17,1,13,1,5,1,114,55,0,0,0,105,32,13,0, 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, @@ -368,7 +368,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, - 117,114,99,101,243,0,0,0,115,46,0,0,0,0,18,12, + 117,114,99,101,244,0,0,0,115,46,0,0,0,0,18,12, 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, @@ -447,7 +447,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, - 99,97,99,104,101,31,1,0,0,115,44,0,0,0,0,9, + 99,97,99,104,101,32,1,0,0,115,44,0,0,0,0,9, 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, - 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,64, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,65, 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, @@ -499,7 +499,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, - 101,116,95,99,97,99,104,101,100,83,1,0,0,115,16,0, + 101,116,95,99,97,99,104,101,100,84,1,0,0,115,16,0, 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, @@ -514,7 +514,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, - 99,95,109,111,100,101,95,1,0,0,115,12,0,0,0,0, + 99,95,109,111,100,101,96,1,0,0,115,12,0,0,0,0, 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, @@ -554,7 +554,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,115,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,116,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -573,7 +573,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,126,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,127,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, @@ -581,7 +581,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,107,1,0,0,115,14,0,0, + 101,99,107,95,110,97,109,101,108,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, @@ -611,7 +611,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,135,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 109,136,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, @@ -698,7 +698,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, - 95,104,101,97,100,101,114,152,1,0,0,115,76,0,0,0, + 95,104,101,97,100,101,114,153,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, @@ -729,7 +729,7 @@ const unsigned char _Py_M__importlib_external[] = { 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, - 112,105,108,101,95,98,121,116,101,99,111,100,101,207,1,0, + 112,105,108,101,95,98,121,116,101,99,111,100,101,208,1,0, 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -749,7 +749,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, - 101,95,116,111,95,98,121,116,101,99,111,100,101,219,1,0, + 101,95,116,111,95,98,121,116,101,99,111,100,101,220,1,0, 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, @@ -778,7 +778,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, - 100,101,95,115,111,117,114,99,101,229,1,0,0,115,10,0, + 100,101,95,115,111,117,114,99,101,230,1,0,0,115,10,0, 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, @@ -843,7 +843,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, - 108,101,95,108,111,99,97,116,105,111,110,246,1,0,0,115, + 108,101,95,108,111,99,97,116,105,111,110,247,1,0,0,115, 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, @@ -883,7 +883,7 @@ const unsigned char _Py_M__importlib_external[] = { 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, - 111,112,101,110,95,114,101,103,105,115,116,114,121,68,2,0, + 111,112,101,110,95,114,101,103,105,115,116,114,121,69,2,0, 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, @@ -910,7 +910,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, - 99,104,95,114,101,103,105,115,116,114,121,75,2,0,0,115, + 99,104,95,114,101,103,105,115,116,114,121,76,2,0,0,115, 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, @@ -935,7 +935,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, - 99,90,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 99,91,2,0,0,115,26,0,0,0,0,2,15,1,12,1, 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, @@ -954,7 +954,7 @@ const unsigned char _Py_M__importlib_external[] = { 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 102,105,110,100,95,109,111,100,117,108,101,106,2,0,0,115, + 102,105,110,100,95,109,111,100,117,108,101,107,2,0,0,115, 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, @@ -963,7 +963,7 @@ const unsigned char _Py_M__importlib_external[] = { 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,162,0,0,0,56,2, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,57,2, 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, @@ -1001,7 +1001,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,153,0,0,0,125,2,0,0, + 0,0,114,5,0,0,0,114,153,0,0,0,126,2,0,0, 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1012,7 +1012,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, - 117,108,101,133,2,0,0,115,0,0,0,0,122,27,95,76, + 117,108,101,134,2,0,0,115,0,0,0,0,122,27,95,76, 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, @@ -1033,7 +1033,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, - 95,109,111,100,117,108,101,136,2,0,0,115,10,0,0,0, + 95,109,111,100,117,108,101,137,2,0,0,115,10,0,0,0, 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1043,13 +1043,13 @@ const unsigned char _Py_M__importlib_external[] = { 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, - 100,117,108,101,144,2,0,0,115,2,0,0,0,0,1,122, + 100,117,108,101,145,2,0,0,115,2,0,0,0,0,1,122, 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,178,0,0,0,120,2, + 4,0,0,0,114,5,0,0,0,114,178,0,0,0,121,2, 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, @@ -1077,7 +1077,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,109,116,105,109,101,150,2,0,0,115,2,0,0,0, + 104,95,109,116,105,109,101,151,2,0,0,115,2,0,0,0, 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, @@ -1112,7 +1112,7 @@ const unsigned char _Py_M__importlib_external[] = { 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,10,112,97,116,104,95,115,116,97,116,115,158,2,0,0, + 218,10,112,97,116,104,95,115,116,97,116,115,159,2,0,0, 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -1136,7 +1136,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, - 104,101,95,98,121,116,101,99,111,100,101,171,2,0,0,115, + 104,101,95,98,121,116,101,99,111,100,101,172,2,0,0,115, 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, @@ -1153,7 +1153,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,192,0,0,0,181,2,0,0,115,0,0,0,0, + 0,0,114,192,0,0,0,182,2,0,0,115,0,0,0,0, 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, @@ -1175,7 +1175,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, - 95,115,111,117,114,99,101,188,2,0,0,115,14,0,0,0, + 95,115,111,117,114,99,101,189,2,0,0,115,14,0,0,0, 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, @@ -1197,7 +1197,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, - 117,114,99,101,95,116,111,95,99,111,100,101,198,2,0,0, + 117,114,99,101,95,116,111,95,99,111,100,101,199,2,0,0, 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, @@ -1259,7 +1259,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 181,0,0,0,206,2,0,0,115,78,0,0,0,0,7,15, + 181,0,0,0,207,2,0,0,115,78,0,0,0,0,7,15, 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, @@ -1271,7 +1271,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,188,0,0,0,148,2,0,0,115,14,0,0,0,12,2, + 114,188,0,0,0,149,2,0,0,115,14,0,0,0,12,2, 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, @@ -1300,7 +1300,7 @@ const unsigned char _Py_M__importlib_external[] = { 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, - 0,0,0,7,3,0,0,115,4,0,0,0,0,3,9,1, + 0,0,0,8,3,0,0,115,4,0,0,0,0,3,9,1, 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, @@ -1309,7 +1309,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,95,101,113,95,95,13,3,0,0,115, + 0,0,0,218,6,95,95,101,113,95,95,14,3,0,0,115, 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, @@ -1318,7 +1318,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, - 115,104,95,95,17,3,0,0,115,2,0,0,0,0,1,122, + 115,104,95,95,18,3,0,0,115,2,0,0,0,0,1,122, 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, @@ -1333,7 +1333,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, - 0,20,3,0,0,115,2,0,0,0,0,10,122,22,70,105, + 0,21,3,0,0,115,2,0,0,0,0,10,122,22,70,105, 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, @@ -1343,7 +1343,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,151,0,0,0,32,3,0,0,115,2, + 114,5,0,0,0,114,151,0,0,0,33,3,0,0,115,2, 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, @@ -1356,14 +1356,14 @@ const unsigned char _Py_M__importlib_external[] = { 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,194,0,0,0,37,3,0,0,115,4,0,0, + 0,0,0,114,194,0,0,0,38,3,0,0,115,4,0,0, 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, - 205,0,0,0,114,5,0,0,0,114,204,0,0,0,2,3, + 205,0,0,0,114,5,0,0,0,114,204,0,0,0,3,3, 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, @@ -1387,7 +1387,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, - 0,0,47,3,0,0,115,4,0,0,0,0,2,12,1,122, + 0,0,48,3,0,0,115,4,0,0,0,0,2,12,1,122, 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, @@ -1397,7 +1397,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,52,3,0, + 0,0,0,114,5,0,0,0,114,193,0,0,0,53,3,0, 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, @@ -1436,7 +1436,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,192,0,0,0,57,3,0,0,115, + 0,114,5,0,0,0,114,192,0,0,0,58,3,0,0,115, 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, @@ -1445,7 +1445,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,212,0,0,0,43,3,0,0,115,8,0,0, + 0,0,0,114,212,0,0,0,44,3,0,0,115,8,0,0, 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, @@ -1467,7 +1467,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,92,3,0,0,115,8, + 114,5,0,0,0,114,181,0,0,0,93,3,0,0,115,8, 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, @@ -1477,13 +1477,13 @@ const unsigned char _Py_M__importlib_external[] = { 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,98,3,0,0,115,2,0, + 5,0,0,0,114,196,0,0,0,99,3,0,0,115,2,0, 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,88, + 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,89, 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, @@ -1508,7 +1508,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,115, + 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,116, 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, @@ -1518,7 +1518,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,207,0,0,0,119,3,0,0,115,4,0,0,0,0,1, + 114,207,0,0,0,120,3,0,0,115,4,0,0,0,0,1, 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, @@ -1527,7 +1527,7 @@ const unsigned char _Py_M__importlib_external[] = { 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, - 0,123,3,0,0,115,2,0,0,0,0,1,122,28,69,120, + 0,124,3,0,0,115,2,0,0,0,0,1,122,28,69,120, 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, @@ -1544,7 +1544,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,126,3,0, + 0,0,0,114,5,0,0,0,114,180,0,0,0,127,3,0, 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, @@ -1562,7 +1562,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,185,0,0,0,134,3,0,0,115,6,0,0, + 0,0,0,114,185,0,0,0,135,3,0,0,115,6,0,0, 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, @@ -1581,7 +1581,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, - 112,114,62,143,3,0,0,115,2,0,0,0,6,1,122,49, + 112,114,62,144,3,0,0,115,2,0,0,0,6,1,122,49, 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, @@ -1589,7 +1589,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, - 0,0,0,114,153,0,0,0,140,3,0,0,115,6,0,0, + 0,0,0,114,153,0,0,0,141,3,0,0,115,6,0,0, 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1600,7 +1600,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,181,0,0,0,146,3, + 4,0,0,0,114,5,0,0,0,114,181,0,0,0,147,3, 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, @@ -1611,7 +1611,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,150,3,0,0,115,2,0,0,0,0,2,122, + 196,0,0,0,151,3,0,0,115,2,0,0,0,0,2,122, 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, @@ -1622,7 +1622,7 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,154,3,0,0,115,2,0,0,0,0,3, + 114,151,0,0,0,155,3,0,0,115,2,0,0,0,0,3, 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, @@ -1631,7 +1631,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 218,0,0,0,107,3,0,0,115,20,0,0,0,12,6,6, + 218,0,0,0,108,3,0,0,115,20,0,0,0,12,6,6, 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, @@ -1675,7 +1675,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,167,3,0,0,115,8, + 114,5,0,0,0,114,179,0,0,0,168,3,0,0,115,8, 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, @@ -1694,7 +1694,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, - 97,116,104,95,110,97,109,101,115,173,3,0,0,115,8,0, + 97,116,104,95,110,97,109,101,115,174,3,0,0,115,8,0, 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, @@ -1707,7 +1707,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,227,0,0,0,183,3, + 4,0,0,0,114,5,0,0,0,114,227,0,0,0,184,3, 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, @@ -1725,7 +1725,7 @@ const unsigned char _Py_M__importlib_external[] = { 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,187, + 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,188, 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, @@ -1734,7 +1734,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,200, + 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,201, 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, @@ -1742,7 +1742,7 @@ const unsigned char _Py_M__importlib_external[] = { 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,95,108,101,110,95,95,203,3,0,0,115, + 0,0,218,7,95,95,108,101,110,95,95,204,3,0,0,115, 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, @@ -1751,7 +1751,7 @@ const unsigned char _Py_M__importlib_external[] = { 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,95,114,101,112,114,95,95,206,3,0, + 0,0,0,218,8,95,95,114,101,112,114,95,95,207,3,0, 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, @@ -1759,7 +1759,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,209, + 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,210, 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, @@ -1768,7 +1768,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 157,0,0,0,212,3,0,0,115,2,0,0,0,0,1,122, + 157,0,0,0,213,3,0,0,115,2,0,0,0,0,1,122, 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, @@ -1776,7 +1776,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, - 0,0,0,160,3,0,0,115,20,0,0,0,12,5,6,2, + 0,0,0,161,3,0,0,115,20,0,0,0,12,5,6,2, 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, @@ -1795,7 +1795,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,218,3,0,0,115,2,0,0,0,0,1, + 114,179,0,0,0,219,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, @@ -1812,21 +1812,21 @@ const unsigned char _Py_M__importlib_external[] = { 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 109,111,100,117,108,101,95,114,101,112,114,221,3,0,0,115, + 109,111,100,117,108,101,95,114,101,112,114,222,3,0,0,115, 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,153,0,0,0,230,3,0,0,115, + 0,114,5,0,0,0,114,153,0,0,0,231,3,0,0,115, 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,196,0,0,0,233,3, + 4,0,0,0,114,5,0,0,0,114,196,0,0,0,234,3, 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, @@ -1836,7 +1836,7 @@ const unsigned char _Py_M__importlib_external[] = { 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,236,3,0,0,115,2, + 114,5,0,0,0,114,181,0,0,0,237,3,0,0,115,2, 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, @@ -1846,14 +1846,14 @@ const unsigned char _Py_M__importlib_external[] = { 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, - 0,0,239,3,0,0,115,0,0,0,0,122,30,95,78,97, + 0,0,240,3,0,0,115,0,0,0,0,122,30,95,78,97, 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, - 0,242,3,0,0,115,2,0,0,0,0,1,122,28,95,78, + 0,243,3,0,0,115,2,0,0,0,0,1,122,28,95,78, 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, @@ -1871,7 +1871,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,187,0,0,0,245,3,0,0,115, + 0,114,5,0,0,0,114,187,0,0,0,246,3,0,0,115, 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, @@ -1880,7 +1880,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, - 217,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, + 218,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, @@ -1917,7 +1917,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,244,0,0,0,7,4,0, + 0,0,0,114,5,0,0,0,114,244,0,0,0,8,4,0, 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, @@ -1943,7 +1943,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, - 111,107,115,15,4,0,0,115,16,0,0,0,0,7,25,1, + 111,107,115,16,4,0,0,115,16,0,0,0,0,7,25,1, 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, @@ -1975,7 +1975,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,32,4, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,33,4, 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, @@ -1995,7 +1995,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, - 103,97,99,121,95,103,101,116,95,115,112,101,99,54,4,0, + 103,97,99,121,95,103,101,116,95,115,112,101,99,55,4,0, 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, @@ -2031,7 +2031,7 @@ const unsigned char _Py_M__importlib_external[] = { 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,9,95,103,101,116,95,115,112,101,99,69,4,0,0, + 0,218,9,95,103,101,116,95,115,112,101,99,70,4,0,0, 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, @@ -2059,7 +2059,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 175,0,0,0,101,4,0,0,115,26,0,0,0,0,4,12, + 175,0,0,0,102,4,0,0,115,26,0,0,0,0,4,12, 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, @@ -2081,7 +2081,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,176,0,0,0,123,4,0,0,115,8,0,0,0,0,8, + 114,176,0,0,0,124,4,0,0,115,8,0,0,0,0,8, 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, @@ -2089,7 +2089,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,243,0,0,0,3,4,0,0,115,22,0,0,0,12,2, + 114,243,0,0,0,4,4,0,0,115,22,0,0,0,12,2, 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, @@ -2138,7 +2138,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,152,4,0,0,115,2,0,0,0,6,0, + 114,221,0,0,0,153,4,0,0,115,2,0,0,0,6,0, 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, @@ -2151,7 +2151,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, - 0,0,146,4,0,0,115,16,0,0,0,0,4,6,1,19, + 0,0,147,4,0,0,115,16,0,0,0,0,4,6,1,19, 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, @@ -2161,7 +2161,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,244,0,0,0,160,4,0,0,115,2,0,0,0, + 0,0,114,244,0,0,0,161,4,0,0,115,2,0,0,0, 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, @@ -2185,7 +2185,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,117,0,0,0,166,4,0,0,115,8,0,0,0,0,7, + 114,117,0,0,0,167,4,0,0,115,8,0,0,0,0,7, 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, @@ -2196,7 +2196,7 @@ const unsigned char _Py_M__importlib_external[] = { 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,255,0,0,0,178,4, + 4,0,0,0,114,5,0,0,0,114,255,0,0,0,179,4, 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, @@ -2260,7 +2260,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,175,0,0,0,183,4,0,0,115,70,0,0,0,0,3, + 114,175,0,0,0,184,4,0,0,115,70,0,0,0,0,3, 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, @@ -2296,7 +2296,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,2, + 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,3, 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, @@ -2314,7 +2314,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, - 229,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, + 230,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, @@ -2352,7 +2352,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, - 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,14, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,15, 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, @@ -2360,7 +2360,7 @@ const unsigned char _Py_M__importlib_external[] = { 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, - 0,0,218,9,112,97,116,104,95,104,111,111,107,4,5,0, + 0,0,218,9,112,97,116,104,95,104,111,111,107,5,5,0, 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, @@ -2369,7 +2369,7 @@ const unsigned char _Py_M__importlib_external[] = { 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,238,0,0,0,22,5,0,0,115,2,0,0,0,0, + 0,114,238,0,0,0,23,5,0,0,115,2,0,0,0,0, 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, @@ -2377,7 +2377,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,0,1,0,0,137,4,0,0, + 0,0,114,5,0,0,0,114,0,1,0,0,138,4,0,0, 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, @@ -2403,7 +2403,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, - 95,102,105,120,95,117,112,95,109,111,100,117,108,101,28,5, + 95,102,105,120,95,117,112,95,109,111,100,117,108,101,29,5, 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, @@ -2424,7 +2424,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,155,0,0,0,51,5,0,0,115,8, + 114,5,0,0,0,114,155,0,0,0,52,5,0,0,115,8, 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, @@ -2486,7 +2486,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,87,5,0,0,115,2,0,0,0,6,0, + 114,221,0,0,0,88,5,0,0,115,2,0,0,0,6,0, 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, @@ -2516,7 +2516,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,115,101,116,117,112,62,5,0,0,115, + 0,0,0,218,6,95,115,101,116,117,112,63,5,0,0,115, 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, @@ -2542,7 +2542,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,105,110,115,116,97,108,108,130,5,0, + 0,0,0,218,8,95,105,110,115,116,97,108,108,131,5,0, 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, @@ -2571,7 +2571,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, - 12,9,12,5,12,7,15,22,15,110,22,1,18,2,6,1, + 12,9,12,5,12,7,15,22,15,111,22,1,18,2,6,1, 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 19259e1..3872256 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -154,7 +154,7 @@ static void *opcode_targets[256] = { &&TARGET_BUILD_TUPLE_UNPACK, &&TARGET_BUILD_SET_UNPACK, &&TARGET_SETUP_ASYNC_WITH, - &&_unknown_opcode, + &&TARGET_FORMAT_VALUE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, -- cgit v0.12 From 281d5321a3bdee1fd7091e439855c0358371216c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 13:09:01 -0500 Subject: Issue 25483: Update dis.rst with FORMAT_VALUE opcode description. --- Doc/library/dis.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1bcb3a4..c6e1656 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -989,6 +989,25 @@ the more significant byte last. arguments. +.. opcode:: FORMAT_VALUE (flags) + + Used for implementing formatted literal strings (f-strings). Pops + an optional *fmt_spec* from the stack, then a required *value*. + *flags* is interpreted as follows: + + * ``(flags & 0x03) == 0x00``: *value* is formattedd as-is. + * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before + formatting it. + * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before + formatting it. + * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before + formatting it. + * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use + it, else use an empty *fmt_spec*. + + Formatting is performed using the :c:func:`PyObject_Format` function. + + .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between -- cgit v0.12 From 9ce52e3bdac2b393c8de5b0da3b1a537dec1928c Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 3 Nov 2015 16:30:49 -0500 Subject: Issue 25483: Fix doc typo and added versionadded. Thanks, Berker Peksag. --- Doc/library/dis.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index c6e1656..ef601e6 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -995,7 +995,7 @@ the more significant byte last. an optional *fmt_spec* from the stack, then a required *value*. *flags* is interpreted as follows: - * ``(flags & 0x03) == 0x00``: *value* is formattedd as-is. + * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before formatting it. * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before @@ -1007,6 +1007,8 @@ the more significant byte last. Formatting is performed using the :c:func:`PyObject_Format` function. + .. versionadded:: 3.6 + .. opcode:: HAVE_ARGUMENT -- cgit v0.12 From 4a91d2138199d28cd906997854cc4632f156bb05 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 3 Nov 2015 22:00:26 -0500 Subject: Neaten-up the inner-loop logic. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 1ca6c72..0e59470 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1053,13 +1053,13 @@ deque_index(dequeobject *deque, PyObject *args) } } - n = stop - i; - while (n--) { + n = stop - i + 1; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); if (cmp > 0) - return PyLong_FromSsize_t(stop - (n + 1)); + return PyLong_FromSsize_t(stop - n); if (cmp < 0) return NULL; if (start_state != deque->state) { -- cgit v0.12 From 45550178ef6102aad81b75462df828e663f3c640 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2015 09:03:53 +0100 Subject: regrtest: display progress every 30 seconds (instead of 60 seconds) when running tests in multiprocessing mode (-jN). --- Lib/test/libregrtest/runtest_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 4473db4..0ca7dd7 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,7 +21,7 @@ from test.libregrtest.setup import setup_tests PROGRESS_MIN_TIME = 30.0 # seconds # Display the running tests if nothing happened last N seconds -PROGRESS_UPDATE = 60.0 # seconds +PROGRESS_UPDATE = 30.0 # seconds def run_test_in_subprocess(testname, ns): -- cgit v0.12 From a3a3d732ec951e272a763f4539439f67d42ef38e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Wed, 4 Nov 2015 07:11:13 -0500 Subject: For FORMAT_VALUE opcode, make it clear that the result of PyObject_Format is pushed on the stack. --- Doc/library/dis.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index ef601e6..7222636 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1005,7 +1005,8 @@ the more significant byte last. * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use it, else use an empty *fmt_spec*. - Formatting is performed using the :c:func:`PyObject_Format` function. + Formatting is performed using :c:func:`PyObject_Format`. The + result is pushed on the stack. .. versionadded:: 3.6 -- cgit v0.12 From cf01b68b889f637c6fb3f11029df3a96b840333d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2015 11:21:38 +0100 Subject: sysmodule.c: reuse Py_STRINGIFY() macro --- Python/sysmodule.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 334f5d0..e0aa233 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1643,15 +1643,11 @@ make_version_info(void) /* sys.implementation values */ #define NAME "cpython" const char *_PySys_ImplName = NAME; -#define QUOTE(arg) #arg -#define STRIFY(name) QUOTE(name) -#define MAJOR STRIFY(PY_MAJOR_VERSION) -#define MINOR STRIFY(PY_MINOR_VERSION) +#define MAJOR Py_STRINGIFY(PY_MAJOR_VERSION) +#define MINOR Py_STRINGIFY(PY_MINOR_VERSION) #define TAG NAME "-" MAJOR MINOR const char *_PySys_ImplCacheTag = TAG; #undef NAME -#undef QUOTE -#undef STRIFY #undef MAJOR #undef MINOR #undef TAG -- cgit v0.12 From e20310fa19188f764b84fbdc40d32d6933f98af9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2015 13:56:58 +0100 Subject: Issue #25556: Add assertions to PyObject_GetItem() to ensure that an exception is raised when it returns NULL. Simplify also ceval.c: rely on the fact that PyObject_GetItem() raised an exception when it returns NULL. --- Objects/abstract.c | 11 ++++++++--- Python/ceval.c | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Objects/abstract.c b/Objects/abstract.c index 63fcf15..2c1c76e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -141,8 +141,11 @@ PyObject_GetItem(PyObject *o, PyObject *key) return null_error(); m = o->ob_type->tp_as_mapping; - if (m && m->mp_subscript) - return m->mp_subscript(o, key); + if (m && m->mp_subscript) { + PyObject *item = m->mp_subscript(o, key); + assert((item != NULL) ^ (PyErr_Occurred() != NULL)); + return item; + } if (o->ob_type->tp_as_sequence) { if (PyIndex_Check(key)) { @@ -1526,8 +1529,10 @@ PySequence_GetItem(PyObject *s, Py_ssize_t i) if (i < 0) { if (m->sq_length) { Py_ssize_t l = (*m->sq_length)(s); - if (l < 0) + if (l < 0) { + assert(PyErr_Occurred()); return NULL; + } i += l; } } diff --git a/Python/ceval.c b/Python/ceval.c index 67ea388..7f9ef6f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2307,7 +2307,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { v = PyObject_GetItem(locals, name); - if (v == NULL && _PyErr_OCCURRED()) { + if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); @@ -2426,7 +2426,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) } else { value = PyObject_GetItem(locals, name); - if (value == NULL && PyErr_Occurred()) { + if (value == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); -- cgit v0.12 From 12b2538ab8e44d69d9ed2c8b329812130db9e6bc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 Nov 2015 17:43:42 +0200 Subject: Reuse Py_STRINGIFY() macro in sre_lib.h and dynload_win.c. --- Modules/sre_lib.h | 6 ++---- Python/dynload_win.c | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index 6ad2ab7..78f7ac7 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -367,14 +367,12 @@ SRE(info)(SRE_STATE* state, SRE_CODE* pattern) #define RETURN_ON_FAILURE(i) \ do { RETURN_ON_ERROR(i); if (i == 0) RETURN_FAILURE; } while (0) -#define SFY(x) #x - #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ "(%" PY_FORMAT_SIZE_T "d)\n", \ - SFY(type), alloc_pos, sizeof(type))); \ + Py_STRINGIFY(type), alloc_pos, sizeof(type))); \ if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ if (j < 0) return j; \ @@ -387,7 +385,7 @@ do { \ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", Py_STRINGIFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) diff --git a/Python/dynload_win.c b/Python/dynload_win.c index f2c796e..9fb0525 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -24,12 +24,10 @@ void _Py_DeactivateActCtx(ULONG_PTR cookie); #define PYD_DEBUG_SUFFIX "" #endif -#define STRINGIZE2(x) #x -#define STRINGIZE(x) STRINGIZE2(x) #ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" STRINGIZE(PY_MAJOR_VERSION) STRINGIZE(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) "-" PYD_PLATFORM_TAG ".pyd" #else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" STRINGIZE(PY_MAJOR_VERSION) STRINGIZE(PY_MINOR_VERSION) ".pyd" +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) ".pyd" #endif #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" -- cgit v0.12 From c106c68aeb5814583a622587e63504f0c93c5140 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Nov 2015 17:01:48 +0100 Subject: Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node when compiling AST from Python objects. --- Include/Python-ast.h | 5 +++-- Misc/NEWS | 3 +++ Parser/asdl_c.py | 12 ++++++++++-- Python/Python-ast.c | 31 +++++++++++++++++++++++++++++-- Python/ast.c | 9 +++------ 5 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index ea6679c..175e380 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -602,8 +602,9 @@ excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_seq * arguments_ty _Py_arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * kw_defaults, arg_ty kwarg, asdl_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2) _Py_arg(a0, a1, a2) -arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena); +#define arg(a0, a1, a2, a3, a4) _Py_arg(a0, a1, a2, a3, a4) +arg_ty _Py_arg(identifier arg, expr_ty annotation, int lineno, int col_offset, + PyArena *arena); #define keyword(a0, a1, a2) _Py_keyword(a0, a1, a2) keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) diff --git a/Misc/NEWS b/Misc/NEWS index e42df93..68de958 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25555: Fix parser and AST: fill lineno and col_offset of "arg" node + when compiling AST from Python objects. + - Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that was mutated by direct calls of dict methods. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 3128078..eedd89b 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -275,7 +275,9 @@ class PrototypeVisitor(EmitVisitor): def visitProduct(self, prod, name): self.emit_function(name, get_c_type(name), - self.get_args(prod.fields), [], union=False) + self.get_args(prod.fields), + self.get_args(prod.attributes), + union=False) class FunctionVisitor(PrototypeVisitor): @@ -329,7 +331,8 @@ class FunctionVisitor(PrototypeVisitor): self.emit(s, depth, reflow) for argtype, argname, opt in args: emit("p->%s = %s;" % (argname, argname), 1) - assert not attrs + for argtype, argname, opt in attrs: + emit("p->%s = %s;" % (argname, argname), 1) class PickleVisitor(EmitVisitor): @@ -452,10 +455,15 @@ class Obj2ModVisitor(PickleVisitor): self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: self.visitFieldDeclaration(f, name, prod=prod, depth=1) + for a in prod.attributes: + self.visitFieldDeclaration(a, name, prod=prod, depth=1) self.emit("", 0) for f in prod.fields: self.visitField(f, name, prod=prod, depth=1) + for a in prod.attributes: + self.visitField(a, name, prod=prod, depth=1) args = [f.name for f in prod.fields] + args.extend([a.name for a in prod.attributes]) self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1) self.emit("return 0;", 1) self.emit("failed:", 0) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a2e9816..07d9b3e 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -2425,7 +2425,8 @@ arguments(asdl_seq * args, arg_ty vararg, asdl_seq * kwonlyargs, asdl_seq * } arg_ty -arg(identifier arg, expr_ty annotation, PyArena *arena) +arg(identifier arg, expr_ty annotation, int lineno, int col_offset, PyArena + *arena) { arg_ty p; if (!arg) { @@ -2438,6 +2439,8 @@ arg(identifier arg, expr_ty annotation, PyArena *arena) return NULL; p->arg = arg; p->annotation = annotation; + p->lineno = lineno; + p->col_offset = col_offset; return p; } @@ -7247,6 +7250,8 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) PyObject* tmp = NULL; identifier arg; expr_ty annotation; + int lineno; + int col_offset; if (_PyObject_HasAttrId(obj, &PyId_arg)) { int res; @@ -7269,7 +7274,29 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) } else { annotation = NULL; } - *out = arg(arg, annotation, arena); + if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &lineno, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); + return 1; + } + if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp == NULL) goto failed; + res = obj2ast_int(tmp, &col_offset, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); + return 1; + } + *out = arg(arg, annotation, lineno, col_offset, arena); return 0; failed: Py_XDECREF(tmp); diff --git a/Python/ast.c b/Python/ast.c index 5a7a745..77ebc83 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1181,11 +1181,9 @@ ast_for_arg(struct compiling *c, const node *n) return NULL; } - ret = arg(name, annotation, c->c_arena); + ret = arg(name, annotation, LINENO(n), n->n_col_offset, c->c_arena); if (!ret) return NULL; - ret->lineno = LINENO(n); - ret->col_offset = n->n_col_offset; return ret; } @@ -1241,11 +1239,10 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, goto error; if (forbidden_name(c, argname, ch, 0)) goto error; - arg = arg(argname, annotation, c->c_arena); + arg = arg(argname, annotation, LINENO(ch), ch->n_col_offset, + c->c_arena); if (!arg) goto error; - arg->lineno = LINENO(ch); - arg->col_offset = ch->n_col_offset; asdl_seq_SET(kwonlyargs, j++, arg); i += 2; /* the name and the comma */ break; -- cgit v0.12 From fad85aadb0e168b7bde414694e448f34bb38c8ef Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Nov 2015 15:42:38 +0200 Subject: Issue #25558: Use compile-time asserts. --- Include/pymacro.h | 4 ++++ Modules/_ctypes/_ctypes.c | 2 +- Modules/_datetimemodule.c | 6 +++--- Modules/_pickle.c | 2 +- Modules/pyexpat.c | 3 ++- Python/pytime.c | 25 ++++++++++++------------- Python/random.c | 2 +- 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 3f6f5dc..49929e5 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -36,6 +36,10 @@ #define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#define Py_BUILD_ASSERT(cond) do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) + /* Get the number of elements in a visible array This does not work on pointers, or arrays declared as [], or function diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b9fd82e..ac4323a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2386,7 +2386,7 @@ unique_key(CDataObject *target, Py_ssize_t index) char *cp = string; size_t bytes_left; - assert(sizeof(string) - 1 > sizeof(Py_ssize_t) * 2); + Py_BUILD_ASSERT(sizeof(string) - 1 > sizeof(Py_ssize_t) * 2); cp += sprintf(cp, "%x", Py_SAFE_DOWNCAST(index, Py_ssize_t, int)); while (target->b_base) { bytes_left = sizeof(string) - (cp - string) - 1; diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 94336cf..55988c5 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -5329,19 +5329,19 @@ PyInit__datetime(void) /* A 4-year cycle has an extra leap day over what we'd get from * pasting together 4 single years. */ - assert(DI4Y == 4 * 365 + 1); + Py_BUILD_ASSERT(DI4Y == 4 * 365 + 1); assert(DI4Y == days_before_year(4+1)); /* Similarly, a 400-year cycle has an extra leap day over what we'd * get from pasting together 4 100-year cycles. */ - assert(DI400Y == 4 * DI100Y + 1); + Py_BUILD_ASSERT(DI400Y == 4 * DI100Y + 1); assert(DI400Y == days_before_year(400+1)); /* OTOH, a 100-year cycle has one fewer leap day than we'd get from * pasting together 25 4-year cycles. */ - assert(DI100Y == 25 * DI4Y - 1); + Py_BUILD_ASSERT(DI100Y == 25 * DI4Y - 1); assert(DI100Y == days_before_year(100+1)); one = PyLong_FromLong(1); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0e3a68e..06882d0 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -874,7 +874,7 @@ _write_size64(char *out, size_t value) { size_t i; - assert(sizeof(size_t) <= 8); + Py_BUILD_ASSERT(sizeof(size_t) <= 8); for (i = 0; i < sizeof(size_t); i++) { out[i] = (unsigned char)((value >> (8 * i)) & 0xff); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9a6da73..b45e3da 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -747,7 +747,8 @@ pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyObject *data, s += MAX_CHUNK_SIZE; slen -= MAX_CHUNK_SIZE; } - assert(MAX_CHUNK_SIZE < INT_MAX && slen < INT_MAX); + Py_BUILD_ASSERT(MAX_CHUNK_SIZE <= INT_MAX); + assert(slen <= INT_MAX); rc = XML_Parse(self->itself, s, (int)slen, isfinal); done: diff --git a/Python/pytime.c b/Python/pytime.c index 53611b1..8a6cb55 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -43,7 +43,7 @@ _PyLong_AsTime_t(PyObject *obj) val = PyLong_AsLongLong(obj); #else long val; - assert(sizeof(time_t) <= sizeof(long)); + Py_BUILD_ASSERT(sizeof(time_t) <= sizeof(long)); val = PyLong_AsLong(obj); #endif if (val == -1 && PyErr_Occurred()) { @@ -60,7 +60,7 @@ _PyLong_FromTime_t(time_t t) #if defined(HAVE_LONG_LONG) && SIZEOF_TIME_T == SIZEOF_LONG_LONG return PyLong_FromLongLong((PY_LONG_LONG)t); #else - assert(sizeof(time_t) <= sizeof(long)); + Py_BUILD_ASSERT(sizeof(time_t) <= sizeof(long)); return PyLong_FromLong((long)t); #endif } @@ -209,6 +209,8 @@ _PyTime_FromSeconds(int seconds) /* ensure that integer overflow cannot happen, int type should have 32 bits, whereas _PyTime_t type has at least 64 bits (SEC_TO_MS takes 30 bits). */ + Py_BUILD_ASSERT(INT_MAX <= _PyTime_MAX / SEC_TO_NS); + Py_BUILD_ASSERT(INT_MIN >= _PyTime_MIN / SEC_TO_NS); assert((t >= 0 && t <= _PyTime_MAX / SEC_TO_NS) || (t < 0 && t >= _PyTime_MIN / SEC_TO_NS)); t *= SEC_TO_NS; @@ -219,7 +221,7 @@ _PyTime_t _PyTime_FromNanoseconds(PY_LONG_LONG ns) { _PyTime_t t; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); t = Py_SAFE_DOWNCAST(ns, PY_LONG_LONG, _PyTime_t); return t; } @@ -231,7 +233,7 @@ _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts, int raise) _PyTime_t t; int res = 0; - assert(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(ts->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)ts->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -253,7 +255,7 @@ _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv, int raise) _PyTime_t t; int res = 0; - assert(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(tv->tv_sec) <= sizeof(_PyTime_t)); t = (_PyTime_t)tv->tv_sec; if (_PyTime_check_mul_overflow(t, SEC_TO_NS)) { @@ -304,12 +306,12 @@ _PyTime_FromObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round, else { #ifdef HAVE_LONG_LONG PY_LONG_LONG sec; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); sec = PyLong_AsLongLong(obj); #else long sec; - assert(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) <= sizeof(_PyTime_t)); sec = PyLong_AsLong(obj); #endif @@ -364,10 +366,10 @@ PyObject * _PyTime_AsNanosecondsObject(_PyTime_t t) { #ifdef HAVE_LONG_LONG - assert(sizeof(PY_LONG_LONG) >= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(PY_LONG_LONG) >= sizeof(_PyTime_t)); return PyLong_FromLongLong((PY_LONG_LONG)t); #else - assert(sizeof(long) >= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(long) >= sizeof(_PyTime_t)); return PyLong_FromLong((long)t); #endif } @@ -650,7 +652,7 @@ pymonotonic(_PyTime_t *tp, _Py_clock_info_t *info, int raise) assert(info == NULL || raise); ticks = GetTickCount64(); - assert(sizeof(ticks) <= sizeof(_PyTime_t)); + Py_BUILD_ASSERT(sizeof(ticks) <= sizeof(_PyTime_t)); t = (_PyTime_t)ticks; if (_PyTime_check_mul_overflow(t, MS_TO_NS)) { @@ -774,8 +776,5 @@ _PyTime_Init(void) if (_PyTime_GetMonotonicClockWithInfo(&t, NULL) < 0) return -1; - /* check that _PyTime_FromSeconds() cannot overflow */ - assert(INT_MAX <= _PyTime_MAX / SEC_TO_NS); - assert(INT_MIN >= _PyTime_MIN / SEC_TO_NS); return 0; } diff --git a/Python/random.c b/Python/random.c index 772bfef..9b42d54 100644 --- a/Python/random.c +++ b/Python/random.c @@ -379,7 +379,7 @@ _PyRandom_Init(void) char *env; unsigned char *secret = (unsigned char *)&_Py_HashSecret.uc; Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); - assert(secret_size == sizeof(_Py_HashSecret.uc)); + Py_BUILD_ASSERT(sizeof(_Py_HashSecret_t) == sizeof(_Py_HashSecret.uc)); if (_Py_HashSecret_Initialized) return; -- cgit v0.12 From 29a2f7c6b38e5a6ed891aa72af38974a1ff2d372 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 7 Nov 2015 18:06:24 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. --- Lib/tkinter/test/test_tkinter/test_widgets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/tkinter/test/test_tkinter/test_widgets.py b/Lib/tkinter/test/test_tkinter/test_widgets.py index 7171667..5a01a5a 100644 --- a/Lib/tkinter/test/test_tkinter/test_widgets.py +++ b/Lib/tkinter/test/test_tkinter/test_widgets.py @@ -91,9 +91,10 @@ class ToplevelTest(AbstractToplevelTest, unittest.TestCase): widget = self.create() self.assertEqual(widget['use'], '') parent = self.create(container=True) - wid = parent.winfo_id() - widget2 = self.create(use=wid) - self.assertEqual(int(widget2['use']), wid) + wid = hex(parent.winfo_id()) + with self.subTest(wid=wid): + widget2 = self.create(use=wid) + self.assertEqual(widget2['use'], wid) @add_standard_options(StandardOptionsTests) -- cgit v0.12 From b9d672491d5082c541bf267eb7bb99fdc6529324 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 9 Nov 2015 14:43:31 +0200 Subject: Issue #25263: Trying to fix test_use on Windows. Avoid possible weird behavior of WideInt convertion. "winfo id" always returns string hexadecimal representation. --- Lib/tkinter/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 12085a9..46f86f9 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -845,8 +845,7 @@ class Misc: self.tk.call('winfo', 'height', self._w)) def winfo_id(self): """Return identifier ID for this widget.""" - return self.tk.getint( - self.tk.call('winfo', 'id', self._w)) + return int(self.tk.call('winfo', 'id', self._w), 0) def winfo_interps(self, displayof=0): """Return the name of all Tcl interpreters for this display.""" args = ('winfo', 'interps') + self._displayof(displayof) -- cgit v0.12 From c826ab06c824933652fec3e902ffa75f13ea5c6a Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 11 Nov 2015 22:41:43 -0600 Subject: Fix typos in whatsnew. Reported on docs@ by Igor Zhun. --- Doc/whatsnew/3.6.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0d387bb..3949dff 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -98,8 +98,8 @@ Improved Modules datetime -------- -:meth:`datetime.stftime ` and -:meth:`date.stftime ` methods now support ISO 8601 date +:meth:`datetime.strftime ` and +:meth:`date.strftime ` methods now support ISO 8601 date directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) -- cgit v0.12 From 8adaec56714c0ca902862d8b6279e1e3c969d664 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 13 Nov 2015 22:17:08 +0000 Subject: Issue #25498: Add NEWS entry for 3.6 --- Misc/NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 96c40b3..ce8aea1 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,10 @@ Core and Builtins Library ------- +- Issue #25498: Fix a crash when garbage-collecting ctypes objects created + by wrapping a memoryview. This was a regression made in 3.5a1. Based + on patch by Eryksun. + - Issue #25584: Added "escape" to the __all__ list in the glob module. - Issue #25584: Fixed recursive glob() with patterns starting with '\*\*'. -- cgit v0.12 From 6fe39266c8eb0b3e3f3544f6ad2214c15449e086 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 13 Nov 2015 23:54:02 +0000 Subject: Issue #25590: Complete attribute names even if they are not yet created --- Doc/whatsnew/3.6.rst | 4 ++++ Lib/rlcompleter.py | 8 +++++--- Lib/test/test_rlcompleter.py | 8 ++++++++ Misc/NEWS | 4 ++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3949dff..89d69cc 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -120,6 +120,10 @@ Private and special attribute names now are omitted unless the prefix starts with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) +Names of most attributes listed by :func:`dir` are now completed. +Previously, names of properties and slots which were not yet created on +an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) + urllib.robotparser ------------------ diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index d368876..02e1fa5 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -160,12 +160,14 @@ class Completer: for word in words: if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): + match = "%s.%s" % (expr, word) try: val = getattr(thisobject, word) except Exception: - continue # Exclude properties that are not set - word = self._callable_postfix(val, "%s.%s" % (expr, word)) - matches.append(word) + pass # Include even if attribute not set + else: + match = self._callable_postfix(val, match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index fee39bc..8ff75c7 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -92,6 +92,14 @@ class TestRlcompleter(unittest.TestCase): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertEqual(f.calls, 1) + def test_uncreated_attr(self): + # Attributes like properties and slots should be completed even when + # they haven't been created on an instance + class Foo: + __slots__ = ("bar",) + completer = rlcompleter.Completer(dict(f=Foo())) + self.assertEqual(completer.complete('f.', 0), 'f.bar') + def test_complete(self): completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') diff --git a/Misc/NEWS b/Misc/NEWS index f95fff0..f00b4e7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,10 @@ Core and Builtins Library ------- +- Issue #25590: In the Readline completer, only call getattr() once per + attribute. Also complete names of attributes such as properties and slots + which are listed by dir() but not yet created on an instance. + - Issue #25498: Fix a crash when garbage-collecting ctypes objects created by wrapping a memoryview. This was a regression made in 3.5a1. Based on patch by Eryksun. -- cgit v0.12 From fad4b600744cde16e952a49af95ad6e520d05ca2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 01:29:17 +0000 Subject: Adjust grammar and punctuation in whatsnew/3.6.rst --- Doc/whatsnew/3.6.rst | 22 +++++++++++----------- Misc/NEWS | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 89d69cc..b2066c5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -98,8 +98,8 @@ Improved Modules datetime -------- -:meth:`datetime.strftime ` and -:meth:`date.strftime ` methods now support ISO 8601 date +The :meth:`datetime.strftime() ` and +:meth:`date.strftime() ` methods now support ISO 8601 date directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) @@ -107,17 +107,17 @@ directives ``%G``, ``%u`` and ``%V``. pickle ------ -Objects that need calling ``__new__`` with keyword arguments, can now be pickled +Objects that need calling ``__new__`` with keyword arguments can now be pickled using :ref:`pickle protocols ` older than protocol version 4. Protocol version 4 already supports this case. (Contributed by Serhiy Storchaka in :issue:`24164`.) -rlcomplete ----------- +rlcompleter +----------- Private and special attribute names now are omitted unless the prefix starts -with underscores. A space or a colon can be added after completed keyword. +with underscores. A space or a colon is added after some completed keywords. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) Names of most attributes listed by :func:`dir` are now completed. @@ -128,7 +128,7 @@ an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) urllib.robotparser ------------------ -:class:`~urllib.robotparser.RobotFileParser` now supports ``Crawl-delay`` and +:class:`~urllib.robotparser.RobotFileParser` now supports the ``Crawl-delay`` and ``Request-rate`` extensions. (Contributed by Nikolay Bogoychev in :issue:`16099`.) @@ -136,18 +136,18 @@ urllib.robotparser Optimizations ============= -* The ASCII decoder is now up to 60 times as fast for error handlers: +* The ASCII decoder is now up to 60 times as fast for error handlers ``surrogateescape``, ``ignore`` and ``replace`` (Contributed by Victor Stinner in :issue:`24870`). * The ASCII and the Latin1 encoders are now up to 3 times as fast for the - error ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). + error handler ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25227`). -* The UTF-8 encoder is now up to 75 times as fast for error handlers: +* The UTF-8 encoder is now up to 75 times as fast for error handlers ``ignore``, ``replace``, ``surrogateescape``, ``surrogatepass`` (Contributed by Victor Stinner in :issue:`25267`). -* The UTF-8 decoder is now up to 15 times as fast for error handlers: +* The UTF-8 decoder is now up to 15 times as fast for error handlers ``ignore``, ``replace`` and ``surrogateescape`` (Contributed by Victor Stinner in :issue:`25301`). diff --git a/Misc/NEWS b/Misc/NEWS index 0a11ebf..8f33a5b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -191,10 +191,10 @@ Library - Issue #25203: Failed readline.set_completer_delims() no longer left the module in inconsistent state. -- Issue #25011: rlcomplete now omits private and special attribute names unless +- Issue #25011: rlcompleter now omits private and special attribute names unless the prefix starts with underscores. -- Issue #25209: rlcomplete now can add a space or a colon after completed keyword. +- Issue #25209: rlcompleter now can add a space or a colon after completed keyword. - Issue #22241: timezone.utc name is now plain 'UTC', not 'UTC-00:00'. -- cgit v0.12 From 63c1ebb67b4716720ad6e807d8f6a19042af83eb Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 08:54:30 +0000 Subject: Issue #25168: Temporary timezone and cache debugging --- Lib/test/datetimetester.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 11deffc..332d94b 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1979,7 +1979,16 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - dt = strptime(dtstr, "%z %Z") + try: + dt = strptime(dtstr, "%z %Z") + except ValueError: + import os + self.fail( + "Issue #25168 strptime() failure info:\n" + f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" + f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" + f"_regex_cache={_strptime._regex_cache!r}\n" + ) self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v0.12 From d226d308a3856e67c654ff3d5924be6d24568388 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 11:47:00 +0000 Subject: Issue #23883: Add test.support.check__all__() and test gettext.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patches by Jacek KoÅ‚odziej. --- Doc/library/test.rst | 42 ++++++++++++++++++++++++++++++ Lib/test/support/__init__.py | 61 ++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_gettext.py | 6 +++++ Lib/test/test_support.py | 22 ++++++++++++++++ Misc/ACKS | 1 + 5 files changed, 132 insertions(+) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 85cab3b..797afa5 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -580,6 +580,48 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.5 +.. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) + + Assert that the ``__all__`` variable of *module* contains all public names. + + The module's public names (its API) are detected automatically + based on whether they match the public name convention and were defined in + *module*. + + The *name_of_module* argument can specify (as a string or tuple thereof) what + module(s) an API could be defined in in order to be detected as a public + API. One case for this is when *module* imports part of its public API from + other modules, possibly a C backend (like ``csv`` and its ``_csv``). + + The *extra* argument can be a set of names that wouldn't otherwise be automatically + detected as "public", like objects without a proper ``__module__`` + attribute. If provided, it will be added to the automatically detected ones. + + The *blacklist* argument can be a set of names that must not be treated as part of + the public API even though their names indicate otherwise. + + Example use:: + + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + .. versionadded:: 3.6 + + The :mod:`test.support` module defines the following classes: .. class:: TransientResource(exc, **kwargs) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 359d6dd..728b459 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -26,6 +26,7 @@ import sys import sysconfig import tempfile import time +import types import unittest import urllib.error import warnings @@ -89,6 +90,7 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", + "check__all__", # sys "is_jython", "check_impl_detail", # network @@ -2199,6 +2201,65 @@ def detect_api_mismatch(ref_api, other_api, *, ignore=()): return missing_items +def check__all__(test_case, module, name_of_module=None, extra=(), + blacklist=()): + """Assert that the __all__ variable of 'module' contains all public names. + + The module's public names (its API) are detected automatically based on + whether they match the public name convention and were defined in + 'module'. + + The 'name_of_module' argument can specify (as a string or tuple thereof) + what module(s) an API could be defined in in order to be detected as a + public API. One case for this is when 'module' imports part of its public + API from other modules, possibly a C backend (like 'csv' and its '_csv'). + + The 'extra' argument can be a set of names that wouldn't otherwise be + automatically detected as "public", like objects without a proper + '__module__' attriubute. If provided, it will be added to the + automatically detected ones. + + The 'blacklist' argument can be a set of names that must not be treated + as part of the public API even though their names indicate otherwise. + + Usage: + import bar + import foo + import unittest + from test import support + + class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, foo) + + class OtherTestCase(unittest.TestCase): + def test__all__(self): + extra = {'BAR_CONST', 'FOO_CONST'} + blacklist = {'baz'} # Undocumented name. + # bar imports part of its API from _bar. + support.check__all__(self, bar, ('bar', '_bar'), + extra=extra, blacklist=blacklist) + + """ + + if name_of_module is None: + name_of_module = (module.__name__, ) + elif isinstance(name_of_module, str): + name_of_module = (name_of_module, ) + + expected = set(extra) + + for name in dir(module): + if name.startswith('_') or name in blacklist: + continue + obj = getattr(module, name) + if (getattr(obj, '__module__', None) in name_of_module or + (not hasattr(obj, '__module__') and + not isinstance(obj, types.ModuleType))): + expected.add(name) + test_case.assertCountEqual(module.__all__, expected) + + class SuppressCrashReport: """Try to prevent a crash report from popping up. diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index de610c7..3a94383 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -440,6 +440,12 @@ class GettextCacheTestCase(GettextBaseTest): self.assertEqual(t.__class__, DummyGNUTranslations) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'c2py', 'ENOENT'} + support.check__all__(self, gettext, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 2c00417..a9ba460 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -312,6 +312,28 @@ class TestSupport(unittest.TestCase): self.OtherClass, self.RefClass, ignore=ignore) self.assertEqual(set(), missing_items) + def test_check__all__(self): + extra = {'tempdir'} + blacklist = {'template'} + support.check__all__(self, + tempfile, + extra=extra, + blacklist=blacklist) + + extra = {'TextTestResult', 'installHandler'} + blacklist = {'load_tests', "TestProgram", "BaseTestSuite"} + + support.check__all__(self, + unittest, + ("unittest.result", "unittest.case", + "unittest.suite", "unittest.loader", + "unittest.main", "unittest.runner", + "unittest.signals"), + extra=extra, + blacklist=blacklist) + + self.assertRaises(AssertionError, support.check__all__, self, unittest) + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled diff --git a/Misc/ACKS b/Misc/ACKS index 6341b23..934cba2 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -765,6 +765,7 @@ Damon Kohler Marko Kohtala Vajrasky Kok Guido Kollerie +Jacek KoÅ‚odziej Jacek Konieczny Марк Коренберг Arkady Koplyarov -- cgit v0.12 From 19e69c5a2067fe6322ead88733ebbca77673010b Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 12:46:42 +0000 Subject: =?UTF-8?q?Issue=20#23883:=20Add=20missing=20APIs=20to=20=5F=5Fall?= =?UTF-8?q?=5F=5F;=20patch=20by=20Jacek=20Ko=C5=82odziej?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/csv.py | 11 ++++++----- Lib/enum.py | 2 +- Lib/ftplib.py | 3 ++- Lib/logging/__init__.py | 5 +++-- Lib/optparse.py | 3 ++- Lib/test/test_csv.py | 6 ++++++ Lib/test/test_enum.py | 7 +++++++ Lib/test/test_ftplib.py | 11 ++++++++++- Lib/test/test_logging.py | 14 +++++++++++++- Lib/test/test_optparse.py | 7 +++++++ Lib/test/test_pickletools.py | 33 +++++++++++++++++++++++++++++++++ Lib/test/test_threading.py | 8 ++++++++ Lib/test/test_wave.py | 7 +++++++ Lib/threading.py | 8 +++++--- Lib/wave.py | 2 +- 15 files changed, 111 insertions(+), 16 deletions(-) diff --git a/Lib/csv.py b/Lib/csv.py index ca40e5e..90461db 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -13,11 +13,12 @@ from _csv import Dialect as _Dialect from io import StringIO -__all__ = [ "QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", - "Error", "Dialect", "__doc__", "excel", "excel_tab", - "field_size_limit", "reader", "writer", - "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter" ] +__all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", + "Error", "Dialect", "__doc__", "excel", "excel_tab", + "field_size_limit", "reader", "writer", + "register_dialect", "get_dialect", "list_dialects", "Sniffer", + "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unix_dialect"] class Dialect: """Describe a CSV dialect. diff --git a/Lib/enum.py b/Lib/enum.py index 8d04e2d..35a9c77 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -8,7 +8,7 @@ except ImportError: from collections import OrderedDict -__all__ = ['Enum', 'IntEnum', 'unique'] +__all__ = ['EnumMeta', 'Enum', 'IntEnum', 'unique'] def _is_descriptor(obj): diff --git a/Lib/ftplib.py b/Lib/ftplib.py index c416d85..2afa19d 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -42,7 +42,8 @@ import socket import warnings from socket import _GLOBAL_DEFAULT_TIMEOUT -__all__ = ["FTP"] +__all__ = ["FTP", "error_reply", "error_temp", "error_perm", "error_proto", + "all_errors"] # Magic number from MSG_OOB = 0x1 # Process data out of band diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 104b0be..369d2c3 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -33,8 +33,9 @@ __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', - 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning', - 'getLogRecordFactory', 'setLogRecordFactory', 'lastResort'] + 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', + 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', + 'lastResort', 'raiseExceptions'] try: import threading diff --git a/Lib/optparse.py b/Lib/optparse.py index 432a2eb..d239ea2 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -38,7 +38,8 @@ __all__ = ['Option', 'OptionError', 'OptionConflictError', 'OptionValueError', - 'BadOptionError'] + 'BadOptionError', + 'check_choice'] __copyright__ = """ Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 8e9c2b4..4763bbb 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1084,5 +1084,11 @@ class TestUnicode(unittest.TestCase): self.assertEqual(fileobj.read(), expected) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {'__doc__', '__version__'} + support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0f7b769..e4e6c2b 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -6,6 +6,7 @@ from collections import OrderedDict from enum import Enum, IntEnum, EnumMeta, unique from io import StringIO from pickle import dumps, loads, PicklingError, HIGHEST_PROTOCOL +from test import support # for pickle tests try: @@ -1708,5 +1709,11 @@ class TestStdLib(unittest.TestCase): if failed: self.fail("result does not equal expected, see print above") + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + support.check__all__(self, enum) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index aef66da..9d8de21 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -1049,10 +1049,19 @@ class TestTimeouts(TestCase): ftp.close() +class MiscTestCase(TestCase): + def test__all__(self): + blacklist = {'MSG_OOB', 'FTP_PORT', 'MAXLINE', 'CRLF', 'B_CRLF', + 'Error', 'parse150', 'parse227', 'parse229', 'parse257', + 'print_line', 'ftpcp', 'test'} + support.check__all__(self, ftplib, blacklist=blacklist) + + def test_main(): tests = [TestFTPClass, TestTimeouts, TestIPv6Environment, - TestTLS_FTPClassMixin, TestTLS_FTPClass] + TestTLS_FTPClassMixin, TestTLS_FTPClass, + MiscTestCase] thread_info = support.threading_setup() try: diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 95575bf..9c4344f 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4159,6 +4159,17 @@ class NTEventLogHandlerTest(BaseTest): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'logThreads', 'logMultiprocessing', + 'logProcesses', 'currentframe', + 'PercentStyle', 'StrFormatStyle', 'StringTemplateStyle', + 'Filterer', 'PlaceHolder', 'Manager', 'RootLogger', + 'root'} + support.check__all__(self, logging, blacklist=blacklist) + + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @@ -4175,7 +4186,8 @@ def test_main(): RotatingFileHandlerTest, LastResortTest, LogRecordTest, ExceptionTest, SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest, - UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest) + UnixSocketHandlerTest, UnixDatagramHandlerTest, UnixSysLogHandlerTest, + MiscTestCase) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7621c24..91a0319 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -16,6 +16,7 @@ from io import StringIO from test import support +import optparse from optparse import make_option, Option, \ TitledHelpFormatter, OptionParser, OptionGroup, \ SUPPRESS_USAGE, OptionError, OptionConflictError, \ @@ -1650,6 +1651,12 @@ class TestParseNumber(BaseTest): "option -l: invalid integer value: '0x12x'") +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'check_builtin', 'AmbiguousOptionError', 'NO_DEFAULT'} + support.check__all__(self, optparse, blacklist=blacklist) + + def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index bbe6875..80221f0 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -4,6 +4,7 @@ import pickletools from test import support from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests +import unittest class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -59,8 +60,40 @@ class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): self.assertNotIn(pickle.BINPUT, pickled2) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'bytes_types', + 'UP_TO_NEWLINE', 'TAKEN_FROM_ARGUMENT1', + 'TAKEN_FROM_ARGUMENT4', 'TAKEN_FROM_ARGUMENT4U', + 'TAKEN_FROM_ARGUMENT8U', 'ArgumentDescriptor', + 'read_uint1', 'read_uint2', 'read_int4', 'read_uint4', + 'read_uint8', 'read_stringnl', 'read_stringnl_noescape', + 'read_stringnl_noescape_pair', 'read_string1', + 'read_string4', 'read_bytes1', 'read_bytes4', + 'read_bytes8', 'read_unicodestringnl', + 'read_unicodestring1', 'read_unicodestring4', + 'read_unicodestring8', 'read_decimalnl_short', + 'read_decimalnl_long', 'read_floatnl', 'read_float8', + 'read_long1', 'read_long4', + 'uint1', 'uint2', 'int4', 'uint4', 'uint8', 'stringnl', + 'stringnl_noescape', 'stringnl_noescape_pair', 'string1', + 'string4', 'bytes1', 'bytes4', 'bytes8', + 'unicodestringnl', 'unicodestring1', 'unicodestring4', + 'unicodestring8', 'decimalnl_short', 'decimalnl_long', + 'floatnl', 'float8', 'long1', 'long4', + 'StackObject', + 'pyint', 'pylong', 'pyinteger_or_bool', 'pybool', 'pyfloat', + 'pybytes_or_str', 'pystring', 'pybytes', 'pyunicode', + 'pynone', 'pytuple', 'pylist', 'pydict', 'pyset', + 'pyfrozenset', 'anyobject', 'markobject', 'stackslice', + 'OpcodeInfo', 'opcodes', 'code2op', + } + support.check__all__(self, pickletools, blacklist=blacklist) + + def test_main(): support.run_unittest(OptimizedPickleTests) + support.run_unittest(MiscTestCase) support.run_doctest(pickletools) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3b11bf6..45564f7 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -18,6 +18,7 @@ import os import subprocess from test import lock_tests +from test import support # Between fork() and exec(), only async-safe functions are allowed (issues @@ -1098,5 +1099,12 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + extra = {"ThreadError"} + blacklist = {'currentThread', 'activeCount'} + support.check__all__(self, threading, ('threading', '_thread'), + extra=extra, blacklist=blacklist) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 3eff773..a67a8b0 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,6 +1,7 @@ from test.support import TESTFN import unittest from test import audiotests +from test import support from audioop import byteswap import sys import wave @@ -103,5 +104,11 @@ class WavePCM32Test(WaveTest, unittest.TestCase): frames = byteswap(frames, 4) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'WAVE_FORMAT_PCM'} + support.check__all__(self, wave, blacklist=blacklist) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/threading.py b/Lib/threading.py index 828019d..2bf33c1 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -22,9 +22,11 @@ except ImportError: # with the multiprocessing module, which doesn't provide the old # Java inspired names. -__all__ = ['active_count', 'Condition', 'current_thread', 'enumerate', 'Event', - 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', - 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size'] +__all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', + 'enumerate', 'main_thread', 'TIMEOUT_MAX', + 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', + 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', + 'setprofile', 'settrace', 'local', 'stack_size'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread diff --git a/Lib/wave.py b/Lib/wave.py index 8a101e3..f71f7e5 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -73,7 +73,7 @@ is destroyed. import builtins -__all__ = ["open", "openfp", "Error"] +__all__ = ["open", "openfp", "Error", "Wave_read", "Wave_write"] class Error(Exception): pass -- cgit v0.12 From 28a465c9e0d4f49eb2f344d8cbb986f61a47efbe Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 14 Nov 2015 12:52:08 +0000 Subject: Issue #23883: Add news listing modules with new exported APIs --- Doc/whatsnew/3.6.rst | 6 ++++++ Misc/NEWS | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b2066c5..172b253 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -230,6 +230,12 @@ Changes in the Python API * The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. +* The following modules have had missing APIs added to their :attr:`__all__` + attributes to match the documented APIs: :mod:`csv`, :mod:`enum`, + :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`threading` and + :mod:`wave`. This means they will export new symbols when ``import *`` + is used. See :issue:`23883`. + Changes in the C API -------------------- diff --git a/Misc/NEWS b/Misc/NEWS index 8f33a5b..4a64eb4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -85,6 +85,11 @@ Core and Builtins Library ------- +- Issue #23883: Added missing APIs to __all__ to match the documented APIs + for the following modules: csv, enum, ftplib, logging, optparse, threading + and wave. Also added a test.support.check__all__() helper. Patches by + Jacek KoÅ‚odziej. + - Issue #25590: In the Readline completer, only call getattr() once per attribute. Also complete names of attributes such as properties and slots which are listed by dir() but not yet created on an instance. -- cgit v0.12 From 413fdcea21908055cb8acad28a94b8f72eb2ffec Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 14 Nov 2015 15:42:17 +0200 Subject: Issue #24821: Refactor STRINGLIB(fastsearch_memchr_1char) and split it on STRINGLIB(find_char) and STRINGLIB(rfind_char) that can be used independedly without special preconditions. --- Objects/bytearrayobject.c | 19 +++--- Objects/bytesobject.c | 19 +++--- Objects/stringlib/fastsearch.h | 150 ++++++++++++++++++++++++----------------- Objects/unicodeobject.c | 33 +++++---- 4 files changed, 121 insertions(+), 100 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 2103147..131e04d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1159,16 +1159,15 @@ bytearray_find_internal(PyByteArrayObject *self, PyObject *args, int dir) ADJUST_INDICES(start, end, len); if (end - start < sub_len) res = -1; - else if (sub_len == 1 -#ifndef HAVE_MEMRCHR - && dir > 0 -#endif - ) { - unsigned char needle = *sub; - int mode = (dir > 0) ? FAST_SEARCH : FAST_RSEARCH; - res = stringlib_fastsearch_memchr_1char( - PyByteArray_AS_STRING(self) + start, end - start, - needle, needle, mode); + else if (sub_len == 1) { + if (dir > 0) + res = stringlib_find_char( + PyByteArray_AS_STRING(self) + start, end - start, + *sub); + else + res = stringlib_rfind_char( + PyByteArray_AS_STRING(self) + start, end - start, + *sub); if (res >= 0) res += start; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index c10dbdf..6a9e062 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1937,16 +1937,15 @@ bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) ADJUST_INDICES(start, end, len); if (end - start < sub_len) res = -1; - else if (sub_len == 1 -#ifndef HAVE_MEMRCHR - && dir > 0 -#endif - ) { - unsigned char needle = *sub; - int mode = (dir > 0) ? FAST_SEARCH : FAST_RSEARCH; - res = stringlib_fastsearch_memchr_1char( - PyBytes_AS_STRING(self) + start, end - start, - needle, needle, mode); + else if (sub_len == 1) { + if (dir > 0) + res = stringlib_find_char( + PyBytes_AS_STRING(self) + start, end - start, + *sub); + else + res = stringlib_rfind_char( + PyBytes_AS_STRING(self) + start, end - start, + *sub); if (res >= 0) res += start; } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index cda68e7..98165ad 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -32,52 +32,98 @@ #define STRINGLIB_BLOOM(mask, ch) \ ((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1))))) - Py_LOCAL_INLINE(Py_ssize_t) -STRINGLIB(fastsearch_memchr_1char)(const STRINGLIB_CHAR* s, Py_ssize_t n, - STRINGLIB_CHAR ch, unsigned char needle, - int mode) +STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) { - if (mode == FAST_SEARCH) { - const STRINGLIB_CHAR *ptr = s; - const STRINGLIB_CHAR *e = s + n; - while (ptr < e) { - void *candidate = memchr((const void *) ptr, needle, (e - ptr) * sizeof(STRINGLIB_CHAR)); - if (candidate == NULL) - return -1; - ptr = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); - if (sizeof(STRINGLIB_CHAR) == 1 || *ptr == ch) - return (ptr - s); - /* False positive */ - ptr++; - } + const STRINGLIB_CHAR *p, *e; + + p = s; + e = s + n; + if (n > 10) { +#if STRINGLIB_SIZEOF_CHAR == 1 + p = memchr(s, ch, n); + if (p != NULL) + return (p - s); return -1; +#else + /* use memchr if we can choose a needle without two many likely + false positives */ + unsigned char needle = ch & 0xff; + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) { + while (p < e) { + void *candidate = memchr(p, needle, + (e - p) * sizeof(STRINGLIB_CHAR)); + if (candidate == NULL) + return -1; + p = (const STRINGLIB_CHAR *) + _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); + if (*p == ch) + return (p - s); + /* False positive */ + p++; + } + return -1; + } +#endif } + while (p < e) { + if (*p == ch) + return (p - s); + p++; + } + return -1; +} + +Py_LOCAL_INLINE(Py_ssize_t) +STRINGLIB(rfind_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) +{ + const STRINGLIB_CHAR *p; #ifdef HAVE_MEMRCHR /* memrchr() is a GNU extension, available since glibc 2.1.91. it doesn't seem as optimized as memchr(), but is still quite - faster than our hand-written loop in FASTSEARCH below */ - else if (mode == FAST_RSEARCH) { - while (n > 0) { - const STRINGLIB_CHAR *found; - void *candidate = memrchr((const void *) s, needle, n * sizeof(STRINGLIB_CHAR)); - if (candidate == NULL) - return -1; - found = (const STRINGLIB_CHAR *) _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); - n = found - s; - if (sizeof(STRINGLIB_CHAR) == 1 || *found == ch) - return n; - /* False positive */ - } + faster than our hand-written loop below */ + + if (n > 10) { +#if STRINGLIB_SIZEOF_CHAR == 1 + p = memrchr(s, ch, n); + if (p != NULL) + return (p - s); return -1; - } +#else + /* use memrchr if we can choose a needle without two many likely + false positives */ + unsigned char needle = ch & 0xff; + /* If looking for a multiple of 256, we'd have too + many false positives looking for the '\0' byte in UCS2 + and UCS4 representations. */ + if (needle != 0) { + while (n > 0) { + void *candidate = memrchr(s, needle, + n * sizeof(STRINGLIB_CHAR)); + if (candidate == NULL) + return -1; + p = (const STRINGLIB_CHAR *) + _Py_ALIGN_DOWN(candidate, sizeof(STRINGLIB_CHAR)); + n = p - s; + if (*p == ch) + return n; + /* False positive */ + } + return -1; + } #endif - else { - assert(0); /* Should never get here */ - return 0; } - -#undef DO_MEMCHR +#endif /* HAVE_MEMRCHR */ + p = s + n; + while (p > s) { + p--; + if (*p == ch) + return (p - s); + } + return -1; } Py_LOCAL_INLINE(Py_ssize_t) @@ -99,25 +145,11 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, if (m <= 0) return -1; /* use special case for 1-character strings */ - if (n > 10 && (mode == FAST_SEARCH -#ifdef HAVE_MEMRCHR - || mode == FAST_RSEARCH -#endif - )) { - /* use memchr if we can choose a needle without two many likely - false positives */ - unsigned char needle; - needle = p[0] & 0xff; -#if STRINGLIB_SIZEOF_CHAR > 1 - /* If looking for a multiple of 256, we'd have too - many false positives looking for the '\0' byte in UCS2 - and UCS4 representations. */ - if (needle != 0) -#endif - return STRINGLIB(fastsearch_memchr_1char) - (s, n, p[0], needle, mode); - } - if (mode == FAST_COUNT) { + if (mode == FAST_SEARCH) + return STRINGLIB(find_char)(s, n, p[0]); + else if (mode == FAST_RSEARCH) + return STRINGLIB(rfind_char)(s, n, p[0]); + else { /* FAST_COUNT */ for (i = 0; i < n; i++) if (s[i] == p[0]) { count++; @@ -125,14 +157,6 @@ FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n, return maxcount; } return count; - } else if (mode == FAST_SEARCH) { - for (i = 0; i < n; i++) - if (s[i] == p[0]) - return i; - } else { /* FAST_RSEARCH */ - for (i = n - 1; i > -1; i--) - if (s[i] == p[0]) - return i; } return -1; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 18a30e2..c2b6f64 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -811,27 +811,26 @@ Py_LOCAL_INLINE(Py_ssize_t) findchar(const void *s, int kind, Py_ssize_t size, Py_UCS4 ch, int direction) { - int mode = (direction == 1) ? FAST_SEARCH : FAST_RSEARCH; - switch (kind) { case PyUnicode_1BYTE_KIND: - { - Py_UCS1 ch1 = (Py_UCS1) ch; - if (ch1 == ch) - return ucs1lib_fastsearch((Py_UCS1 *) s, size, &ch1, 1, 0, mode); - else - return -1; - } + if ((Py_UCS1) ch != ch) + return -1; + if (direction > 0) + return ucs1lib_find_char((Py_UCS1 *) s, size, (Py_UCS1) ch); + else + return ucs1lib_rfind_char((Py_UCS1 *) s, size, (Py_UCS1) ch); case PyUnicode_2BYTE_KIND: - { - Py_UCS2 ch2 = (Py_UCS2) ch; - if (ch2 == ch) - return ucs2lib_fastsearch((Py_UCS2 *) s, size, &ch2, 1, 0, mode); - else - return -1; - } + if ((Py_UCS2) ch != ch) + return -1; + if (direction > 0) + return ucs2lib_find_char((Py_UCS2 *) s, size, (Py_UCS2) ch); + else + return ucs2lib_rfind_char((Py_UCS2 *) s, size, (Py_UCS2) ch); case PyUnicode_4BYTE_KIND: - return ucs4lib_fastsearch((Py_UCS4 *) s, size, &ch, 1, 0, mode); + if (direction > 0) + return ucs4lib_find_char((Py_UCS4 *) s, size, ch); + else + return ucs4lib_rfind_char((Py_UCS4 *) s, size, ch); default: assert(0); return -1; -- cgit v0.12 From fca22327ca9086a17902b3a21f2b8af45f7a4892 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 16 Nov 2015 09:22:19 +0000 Subject: Issue #20220: Revert time zone test debugging, revision 139c18943d9b --- Lib/test/datetimetester.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index f814f23..7374608 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2006,16 +2006,7 @@ class TestDateTime(TestDate): seconds = tzseconds hours, minutes = divmod(seconds//60, 60) dtstr = "{}{:02d}{:02d} {}".format(sign, hours, minutes, tzname) - try: - dt = strptime(dtstr, "%z %Z") - except ValueError: - import os - self.fail( - "Issue #25168 strptime() failure info:\n" - f"_TimeRE_cache['Z']={_strptime._TimeRE_cache['Z']!r}\n" - f"TZ={os.environ.get('TZ')!r}, or {os.getenv('TZ')!r} via getenv()\n" - f"_regex_cache={_strptime._regex_cache!r}\n" - ) + dt = strptime(dtstr, "%z %Z") self.assertEqual(dt.utcoffset(), timedelta(seconds=tzseconds)) self.assertEqual(dt.tzname(), tzname) # Can produce inconsistent datetime -- cgit v0.12 From 9ba97df69c9b056d8531bb8ed54378ca81d761de Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Nov 2015 12:15:07 +0100 Subject: Closes #25645: Fix a reference leak introduced by change bc5894a3a0e6 of the issue #24164. --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 06882d0..a6f414c 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -193,6 +193,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->import_mapping_3to2); Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); + Py_CLEAR(st->partial); } /* Initialize the given pickle module state. */ -- cgit v0.12 From 6019c8ced03fcb7acde715bf39aea18d1e44cba8 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 17 Nov 2015 08:28:07 -0800 Subject: Issue #25629: Move set fill/used updates out of inner loop --- Objects/setobject.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 03bb230..083cbea 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -254,9 +254,10 @@ set_add_entry(PySetObject *so, PyObject *key, Py_hash_t hash) Internal routine used by set_table_resize() to insert an item which is known to be absent from the set. This routine also assumes that the set contains no deleted entries. Besides the performance benefit, -using set_insert_clean() in set_table_resize() is dangerous (SF bug #1456209). -Note that no refcounts are changed by this routine; if needed, the caller -is responsible for incref'ing `key`. +there is also safety benefit since using set_add_entry() risks making +a callback in the middle of a set_table_resize(), see issue 1456209. +The caller is responsible for updating the key's reference count and +the setobject's fill and used fields. */ static void set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) @@ -285,8 +286,6 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) found_null: entry->key = key; entry->hash = hash; - so->fill++; - so->used++; } /* ======== End logic for probing the hash table ========================== */ @@ -356,8 +355,8 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) /* Make the set empty, using the new table. */ assert(newtable != oldtable); memset(newtable, 0, sizeof(setentry) * newsize); - so->fill = 0; - so->used = 0; + so->fill = oldused; + so->used = oldused; so->mask = newsize - 1; so->table = newtable; @@ -676,6 +675,8 @@ set_merge(PySetObject *so, PyObject *otherset) /* If our table is empty, we can use set_insert_clean() */ if (so->fill == 0) { + so->fill = other->used; + so->used = other->used; for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { -- cgit v0.12 From 66f6238fca7286a3917eab0e9fafd7aa6f8e90a4 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 17 Nov 2015 20:58:43 -0800 Subject: Add assertion to verify the pre-condition in the comments. --- Objects/setobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/setobject.c b/Objects/setobject.c index 083cbea..7fde01f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -269,6 +269,7 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) size_t i = (size_t)hash & mask; size_t j; + assert(so->fill == so->used); while (1) { entry = &table[i]; if (entry->key == NULL) -- cgit v0.12 From 8b4ee344ef43552a37c4e6fe342eb4a8c4b1a10d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 20 Nov 2015 02:37:57 +0000 Subject: Issue #25583: Add news to 3.6 section --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 72d2073..1e18892 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,9 @@ Core and Builtins Library ------- +- Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) + when the OS gives priority to errors such as EACCES over EEXIST. + - Issue #25593: Change semantics of EventLoop.stop() in asyncio. - Issue #6973: When we know a subprocess.Popen process has died, do -- cgit v0.12 From b4efc963d628475c79f02ecf43a4ec564b46428e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 20 Nov 2015 09:24:02 +0100 Subject: Issue #25557: Refactor _PyDict_LoadGlobal() Don't fallback to PyDict_GetItemWithError() if the hash is unknown: compute the hash instead. Add also comments to explain the optimization a little bit. --- Objects/dictobject.c | 55 +++++++++++++++++++++++++++------------------------- Python/ceval.c | 11 +++++++++-- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 624ae9b..4a72c9a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1165,39 +1165,42 @@ _PyDict_GetItemIdWithError(PyObject *dp, struct _Py_Identifier *key) return PyDict_GetItemWithError(dp, kv); } -/* Fast version of global value lookup. +/* Fast version of global value lookup (LOAD_GLOBAL). * Lookup in globals, then builtins. + * + * Raise an exception and return NULL if an error occurred (ex: computing the + * key hash failed, key comparison failed, ...). Return NULL if the key doesn't + * exist. Return the value if the key exists. */ PyObject * _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) { - PyObject *x; - if (PyUnicode_CheckExact(key)) { - PyObject **value_addr; - Py_hash_t hash = ((PyASCIIObject *)key)->hash; - if (hash != -1) { - PyDictKeyEntry *e; - e = globals->ma_keys->dk_lookup(globals, key, hash, &value_addr); - if (e == NULL) { - return NULL; - } - x = *value_addr; - if (x != NULL) - return x; - e = builtins->ma_keys->dk_lookup(builtins, key, hash, &value_addr); - if (e == NULL) { - return NULL; - } - x = *value_addr; - return x; - } + Py_hash_t hash; + PyDictKeyEntry *entry; + PyObject **value_addr; + PyObject *value; + + if (!PyUnicode_CheckExact(key) || + (hash = ((PyASCIIObject *) key)->hash) == -1) + { + hash = PyObject_Hash(key); + if (hash == -1) + return NULL; } - x = PyDict_GetItemWithError((PyObject *)globals, key); - if (x != NULL) - return x; - if (PyErr_Occurred()) + + /* namespace 1: globals */ + entry = globals->ma_keys->dk_lookup(globals, key, hash, &value_addr); + if (entry == NULL) return NULL; - return PyDict_GetItemWithError((PyObject *)builtins, key); + value = *value_addr; + if (value != NULL) + return value; + + /* namespace 2: builtins */ + entry = builtins->ma_keys->dk_lookup(builtins, key, hash, &value_addr); + if (entry == NULL) + return NULL; + return *value_addr; } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the diff --git a/Python/ceval.c b/Python/ceval.c index 7f9ef6f..1d65649 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2347,26 +2347,33 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) PyObject *name = GETITEM(names, oparg); PyObject *v; if (PyDict_CheckExact(f->f_globals) - && PyDict_CheckExact(f->f_builtins)) { + && PyDict_CheckExact(f->f_builtins)) + { v = _PyDict_LoadGlobal((PyDictObject *)f->f_globals, (PyDictObject *)f->f_builtins, name); if (v == NULL) { - if (!_PyErr_OCCURRED()) + if (!_PyErr_OCCURRED()) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ format_exc_check_arg(PyExc_NameError, NAME_ERROR_MSG, name); + } goto error; } Py_INCREF(v); } else { /* Slow-path if globals or builtins is not a dict */ + + /* namespace 1: globals */ v = PyObject_GetItem(f->f_globals, name); if (v == NULL) { if (!PyErr_ExceptionMatches(PyExc_KeyError)) goto error; PyErr_Clear(); + /* namespace 2: builtins */ v = PyObject_GetItem(f->f_builtins, name); if (v == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyError)) -- cgit v0.12 From 748dad5b6a54370c72029afea9e7baa683d1ffe8 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Fri, 20 Nov 2015 13:12:26 -0800 Subject: Close 25594: advise against accessing Enum members from other members --- Doc/library/enum.rst | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 0fbbf5a..a76f5a3 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -730,18 +730,24 @@ member instances. Finer Points ^^^^^^^^^^^^ -Enum members are instances of an Enum class, and even though they are -accessible as `EnumClass.member`, they are not accessible directly from -the member:: - - >>> Color.red - - >>> Color.red.blue - Traceback (most recent call last): +:class:`Enum` members are instances of an :class:`Enum` class, and even +though they are accessible as `EnumClass.member`, they should not be accessed +directly from the member as that lookup may fail or, worse, return something +besides the :class:`Enum` member you looking for:: + + >>> class FieldTypes(Enum): + ... name = 0 + ... value = 1 + ... size = 2 ... - AttributeError: 'Color' object has no attribute 'blue' + >>> FieldTypes.value.size + + >>> FieldTypes.size.value + 2 + +.. versionchanged:: 3.5 -Likewise, the :attr:`__members__` is only available on the class. +The :attr:`__members__` attribute is only available on the class. If you give your :class:`Enum` subclass extra methods, like the `Planet`_ class above, those methods will show up in a :func:`dir` of the member, -- cgit v0.12 From 1aa10278a77e76e6d0cacae2d79429c1b3e2c447 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 21 Nov 2015 10:57:47 +0000 Subject: Issue #25626: Add news to 3.6 section --- Misc/NEWS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 0e25ad2..39226f5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,13 @@ Core and Builtins Library ------- +- Issue #25626: Change three zlib functions to accept sizes that fit in + Py_ssize_t, but internally cap those sizes to UINT_MAX. This resolves a + regression in 3.5 where GzipFile.read() failed to read chunks larger than 2 + or 4 GiB. The change affects the zlib.Decompress.decompress() max_length + parameter, the zlib.decompress() bufsize parameter, and the + zlib.Decompress.flush() length parameter. + - Issue #25583: Avoid incorrect errors raised by os.makedirs(exist_ok=True) when the OS gives priority to errors such as EACCES over EEXIST. -- cgit v0.12 From dc0965551e84de92744be587fcb33471a8d565b6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 Nov 2015 15:18:40 +0100 Subject: Issue #25694: Install test.libregrtest to be able to run tests on the installed Python --- Makefile.pre.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 823def3..c6c534e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1167,6 +1167,7 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/cjkencodings test/decimaltestdata test/xmltestdata \ test/eintrdata \ test/imghdrdata \ + test/libregrtest \ test/subprocessdata test/sndhdrdata test/support \ test/tracedmodules test/encoded_modules \ test/test_import \ -- cgit v0.12 From da43ee4316a508d031c2c68aeb0f7797294d09ff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 22 Nov 2015 15:18:54 +0100 Subject: Issue #25694: Fix test_regrtest for installed Python --- Lib/test/test_regrtest.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index ab7741f..59f8c9d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -14,6 +14,7 @@ import platform import re import subprocess import sys +import sysconfig import textwrap import unittest from test import libregrtest @@ -306,7 +307,7 @@ class BaseTestCase(unittest.TestCase): TESTNAME_REGEX = r'test_[a-z0-9_]+' def setUp(self): - self.testdir = os.path.join(ROOT_DIR, 'Lib', 'test') + self.testdir = os.path.realpath(os.path.dirname(__file__)) # When test_regrtest is interrupted by CTRL+c, it can leave # temporary test files @@ -330,8 +331,13 @@ class BaseTestCase(unittest.TestCase): self.addCleanup(support.unlink, path) # Use 'x' mode to ensure that we do not override existing tests - with open(path, 'x', encoding='utf-8') as fp: - fp.write(code) + try: + with open(path, 'x', encoding='utf-8') as fp: + fp.write(code) + except PermissionError as exc: + if not sysconfig.is_python_build(): + self.skipTest("cannot write %s: %s" % (path, exc)) + raise return name def regex_search(self, regex, output): @@ -471,7 +477,7 @@ class ProgramsTestCase(BaseTestCase): def test_script_regrtest(self): # Lib/test/regrtest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'regrtest.py') + script = os.path.join(self.testdir, 'regrtest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) @@ -503,10 +509,12 @@ class ProgramsTestCase(BaseTestCase): def test_script_autotest(self): # Lib/test/autotest.py - script = os.path.join(ROOT_DIR, 'Lib', 'test', 'autotest.py') + script = os.path.join(self.testdir, 'autotest.py') args = [*self.python_args, script, *self.regrtest_args, *self.tests] self.run_tests(args) + @unittest.skipUnless(sysconfig.is_python_build(), + 'run_tests.py script is not installed') def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') @@ -516,6 +524,8 @@ class ProgramsTestCase(BaseTestCase): proc = self.run_command(args) self.check_output(proc.stdout) + @unittest.skipUnless(sysconfig.is_python_build(), + 'test.bat script is not installed') @unittest.skipUnless(sys.platform == 'win32', 'Windows only') def test_tools_buildbot_test(self): # Tools\buildbot\test.bat -- cgit v0.12 From 745f6b3a5db0a9b1cdae7b20c7be2c3c8a3240a7 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 00:20:00 +0000 Subject: Issue #25663: Update rlcompleter test for new 3.6 behaviour --- Lib/test/test_rlcompleter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 34a5eff..d9e4de6 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -124,9 +124,10 @@ class TestRlcompleter(unittest.TestCase): completer = rlcompleter.Completer(namespace) self.assertEqual(completer.complete('False', 0), 'False') self.assertIsNone(completer.complete('False', 1)) # No duplicates - self.assertEqual(completer.complete('assert', 0), 'assert') + # Space or colon added due to being a reserved keyword + self.assertEqual(completer.complete('assert', 0), 'assert ') self.assertIsNone(completer.complete('assert', 1)) - self.assertEqual(completer.complete('try', 0), 'try') + self.assertEqual(completer.complete('try', 0), 'try:') self.assertIsNone(completer.complete('try', 1)) # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') -- cgit v0.12 From 33623b145df4d41f49ab3ae9e781135fb87fd98d Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 22:12:05 +0000 Subject: Issue #25695: Defer creation of TESTDIRN until the test case is run --- Lib/test/test_support.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index a9ba460..f86ea91 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -9,13 +9,11 @@ import errno from test import support TESTFN = support.TESTFN -TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) class TestSupport(unittest.TestCase): def setUp(self): support.unlink(TESTFN) - support.rmtree(TESTDIRN) tearDown = setUp def test_import_module(self): @@ -48,6 +46,10 @@ class TestSupport(unittest.TestCase): support.unlink(TESTFN) def test_rmtree(self): + TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) + self.addCleanup(support.rmtree, TESTDIRN) + support.rmtree(TESTDIRN) + os.mkdir(TESTDIRN) os.mkdir(os.path.join(TESTDIRN, TESTDIRN)) support.rmtree(TESTDIRN) -- cgit v0.12 From f65dd1d4db088ab2a12430f1b6d43e8f1ed04e18 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Tue, 24 Nov 2015 23:00:37 +0000 Subject: Issue #25576: Apply fix to new urlopen() doc string --- Lib/urllib/request.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index e6abf34..57d0dea 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -149,13 +149,8 @@ def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, *data* should be a buffer in the standard application/x-www-form-urlencoded format. The urllib.parse.urlencode() function takes a mapping or sequence - of 2-tuples and returns a string in this format. It should be encoded to - bytes before being used as the data parameter. The charset parameter in - Content-Type header may be used to specify the encoding. If charset - parameter is not sent with the Content-Type header, the server following - the HTTP 1.1 recommendation may assume that the data is encoded in - ISO-8859-1 encoding. It is advisable to use charset parameter with encoding - used in Content-Type header with the Request. + of 2-tuples and returns an ASCII text string in this format. It should be + encoded to bytes before being used as the data parameter. urllib.request module uses HTTP/1.1 and includes a "Connection:close" header in its HTTP requests. -- cgit v0.12 From dde0815c359fc321b0e7a94f885132e2e77534a1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 25 Nov 2015 15:28:13 +0200 Subject: Issue #7990: dir() on ElementTree.Element now lists properties: "tag", "text", "tail" and "attrib". Original patch by Santoso Wijaya. --- Lib/test/test_xml_etree.py | 10 +-- Misc/NEWS | 3 + Modules/_elementtree.c | 170 +++++++++++++++++++++++++-------------------- 3 files changed, 103 insertions(+), 80 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 57d8e4d..0293201 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -182,10 +182,12 @@ class ElementTreeTest(unittest.TestCase): def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") - self.assertTrue(hasattr(element, "tag"), msg="no tag member") - self.assertTrue(hasattr(element, "attrib"), msg="no attrib member") - self.assertTrue(hasattr(element, "text"), msg="no text member") - self.assertTrue(hasattr(element, "tail"), msg="no tail member") + direlem = dir(element) + for attr in 'tag', 'attrib', 'text', 'tail': + self.assertTrue(hasattr(element, attr), + msg='no %s member' % attr) + self.assertIn(attr, direlem, + msg='no %s visible by dir' % attr) check_string(element.tag) check_mapping(element.attrib) diff --git a/Misc/NEWS b/Misc/NEWS index ac1744b..56a4d2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -95,6 +95,9 @@ Core and Builtins Library ------- +- Issue #7990: dir() on ElementTree.Element now lists properties: "tag", + "text", "tail" and "attrib". Original patch by Santoso Wijaya. + - Issue #25725: Fixed a reference leak in pickle.loads() when unpickling invalid data including tuple instructions. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 744e833..f03e136 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1870,94 +1870,92 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) } static PyObject* -element_getattro(ElementObject* self, PyObject* nameobj) +element_tag_getter(ElementObject *self, void *closure) { - PyObject* res; - char *name = ""; + PyObject *res = self->tag; + Py_INCREF(res); + return res; +} - if (PyUnicode_Check(nameobj)) - name = _PyUnicode_AsString(nameobj); +static PyObject* +element_text_getter(ElementObject *self, void *closure) +{ + PyObject *res = element_get_text(self); + Py_XINCREF(res); + return res; +} - if (name == NULL) - return NULL; +static PyObject* +element_tail_getter(ElementObject *self, void *closure) +{ + PyObject *res = element_get_tail(self); + Py_XINCREF(res); + return res; +} - /* handle common attributes first */ - if (strcmp(name, "tag") == 0) { - res = self->tag; - Py_INCREF(res); - return res; - } else if (strcmp(name, "text") == 0) { - res = element_get_text(self); - Py_XINCREF(res); - return res; +static PyObject* +element_attrib_getter(ElementObject *self, void *closure) +{ + PyObject *res; + if (!self->extra) { + if (create_extra(self, NULL) < 0) + return NULL; } + res = element_get_attrib(self); + Py_XINCREF(res); + return res; +} - /* methods */ - res = PyObject_GenericGetAttr((PyObject*) self, nameobj); - if (res) - return res; - - /* less common attributes */ - if (strcmp(name, "tail") == 0) { - PyErr_Clear(); - res = element_get_tail(self); - } else if (strcmp(name, "attrib") == 0) { - PyErr_Clear(); - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return NULL; - } - res = element_get_attrib(self); +/* macro for setter validation */ +#define _VALIDATE_ATTR_VALUE(V) \ + if ((V) == NULL) { \ + PyErr_SetString( \ + PyExc_AttributeError, \ + "can't delete element attribute"); \ + return -1; \ } - if (!res) - return NULL; - - Py_INCREF(res); - return res; +static int +element_tag_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(self->tag); + self->tag = value; + return 0; } static int -element_setattro(ElementObject* self, PyObject* nameobj, PyObject* value) +element_text_setter(ElementObject *self, PyObject *value, void *closure) { - char *name = ""; + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(JOIN_OBJ(self->text)); + self->text = value; + return 0; +} - if (value == NULL) { - PyErr_SetString(PyExc_AttributeError, - "can't delete attribute"); - return -1; - } - if (PyUnicode_Check(nameobj)) - name = _PyUnicode_AsString(nameobj); - if (name == NULL) - return -1; +static int +element_tail_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + Py_INCREF(value); + Py_DECREF(JOIN_OBJ(self->tail)); + self->tail = value; + return 0; +} - if (strcmp(name, "tag") == 0) { - Py_DECREF(self->tag); - self->tag = value; - Py_INCREF(self->tag); - } else if (strcmp(name, "text") == 0) { - Py_DECREF(JOIN_OBJ(self->text)); - self->text = value; - Py_INCREF(self->text); - } else if (strcmp(name, "tail") == 0) { - Py_DECREF(JOIN_OBJ(self->tail)); - self->tail = value; - Py_INCREF(self->tail); - } else if (strcmp(name, "attrib") == 0) { - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return -1; - } - Py_DECREF(self->extra->attrib); - self->extra->attrib = value; - Py_INCREF(self->extra->attrib); - } else { - PyErr_SetString(PyExc_AttributeError, - "Can't set arbitrary attributes on Element"); - return -1; +static int +element_attrib_setter(ElementObject *self, PyObject *value, void *closure) +{ + _VALIDATE_ATTR_VALUE(value); + if (!self->extra) { + if (create_extra(self, NULL) < 0) + return -1; } - + Py_INCREF(value); + Py_DECREF(self->extra->attrib); + self->extra->attrib = value; return 0; } @@ -3770,6 +3768,26 @@ static PyMappingMethods element_as_mapping = { (objobjargproc) element_ass_subscr, }; +static PyGetSetDef element_getsetlist[] = { + {"tag", + (getter)element_tag_getter, + (setter)element_tag_setter, + "A string identifying what kind of data this element represents"}, + {"text", + (getter)element_text_getter, + (setter)element_text_setter, + "A string of text directly after the start tag, or None"}, + {"tail", + (getter)element_tail_getter, + (setter)element_tail_setter, + "A string of text directly after the end tag, or None"}, + {"attrib", + (getter)element_attrib_getter, + (setter)element_attrib_setter, + "A dictionary containing the element's attributes"}, + {NULL}, +}; + static PyTypeObject Element_Type = { PyVarObject_HEAD_INIT(NULL, 0) "xml.etree.ElementTree.Element", sizeof(ElementObject), 0, @@ -3786,8 +3804,8 @@ static PyTypeObject Element_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)element_getattro, /* tp_getattro */ - (setattrofunc)element_setattro, /* tp_setattro */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ @@ -3800,7 +3818,7 @@ static PyTypeObject Element_Type = { 0, /* tp_iternext */ element_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + element_getsetlist, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ -- cgit v0.12 From aa785555b1f8ab2c82b4103c5e9ed2d46af66726 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 26 Nov 2015 02:36:26 +0000 Subject: Issue #25622: Rename to PythonValuesTestCase and enable for non-Windows --- Lib/ctypes/test/test_values.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 9551e7a..c7c78ce 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -28,8 +28,7 @@ class ValuesTestCase(unittest.TestCase): ctdll = CDLL(_ctypes_test.__file__) self.assertRaises(ValueError, c_int.in_dll, ctdll, "Undefined_Symbol") -@unittest.skipUnless(sys.platform == 'win32', 'Windows-specific test') -class Win_ValuesTestCase(unittest.TestCase): +class PythonValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" def test_optimizeflag(self): @@ -76,7 +75,7 @@ class Win_ValuesTestCase(unittest.TestCase): if entry.name in bootstrap_expected: bootstrap_seen.append(entry.name) self.assertTrue(entry.size, - "{} was reported as having no size".format(entry.name)) + "{!r} was reported as having no size".format(entry.name)) continue items.append((entry.name, entry.size)) -- cgit v0.12 From 4f09806e662928c5524ab5d792d73297c50494b3 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 28 Nov 2015 12:24:52 -0500 Subject: #25485: Add context manager support to Telnet class. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Stéphane Wirtel. --- Doc/library/telnetlib.rst | 11 +++++++++++ Doc/whatsnew/3.6.rst | 7 +++++++ Lib/telnetlib.py | 15 ++++++++++----- Lib/test/test_telnetlib.py | 5 +++++ Misc/NEWS | 2 ++ 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst index 4040f72..ce406ca 100644 --- a/Doc/library/telnetlib.rst +++ b/Doc/library/telnetlib.rst @@ -43,6 +43,17 @@ Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). :exc:`EOFError` when the end of the connection is read, because they can return an empty string for other reasons. See the individual descriptions below. + A :class:`Telnet` object is a context manager and can be used in a + :keyword:`with` statement. When the :keyword:`with` block ends, the + :meth:`close` method is called:: + + >>> from telnetlib import Telnet + >>> with Telnet('localhost', 23) as tn: + ... tn.interact() + ... + + .. versionchanged:: 3.6 Context manager support added + .. seealso:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 172b253..fc32fb5 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -125,6 +125,13 @@ Previously, names of properties and slots which were not yet created on an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +telnetlib +--------- + +:class:`~telnetlib.Telnet` is now a context manager (contributed by +Stéphane Wirtel in :issue:`25485`). + + urllib.robotparser ------------------ diff --git a/Lib/telnetlib.py b/Lib/telnetlib.py index 72dabc7..b0863b1 100644 --- a/Lib/telnetlib.py +++ b/Lib/telnetlib.py @@ -637,6 +637,12 @@ class Telnet: raise EOFError return (-1, None, text) + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + def test(): """Test program for telnetlib. @@ -660,11 +666,10 @@ def test(): port = int(portstr) except ValueError: port = socket.getservbyname(portstr, 'tcp') - tn = Telnet() - tn.set_debuglevel(debuglevel) - tn.open(host, port, timeout=0.5) - tn.interact() - tn.close() + with Telnet() as tn: + tn.set_debuglevel(debuglevel) + tn.open(host, port, timeout=0.5) + tn.interact() if __name__ == '__main__': test() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 524bba3..23029e0 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -42,6 +42,11 @@ class GeneralTests(TestCase): telnet = telnetlib.Telnet(HOST, self.port) telnet.sock.close() + def testContextManager(self): + with telnetlib.Telnet(HOST, self.port) as tn: + self.assertIsNotNone(tn.get_socket()) + self.assertIsNone(tn.get_socket()) + def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) diff --git a/Misc/NEWS b/Misc/NEWS index c3cce96..2cb06e9 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #25485: telnetlib.Telnet is now a context manager. + - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside __getattr__. -- cgit v0.12 From 40062a1127a9678669bb0efec89d015bbbed1836 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 28 Nov 2015 22:38:24 +0000 Subject: Issue #25754: Allow test_rlcompleter to be run multiple times --- Lib/test/test_rlcompleter.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d9e4de6..208c054 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import builtins import rlcompleter @@ -72,12 +73,12 @@ class TestRlcompleter(unittest.TestCase): self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) - CompleteMe.me = CompleteMe - self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), - ['CompleteMe.me.me.spam']) - self.assertEqual(self.completer.attr_matches('egg.s'), - ['egg.{}('.format(x) for x in dir(str) - if x.startswith('s')]) + with patch.object(CompleteMe, "me", CompleteMe, create=True): + self.assertEqual(self.completer.attr_matches('CompleteMe.me.me.sp'), + ['CompleteMe.me.me.spam']) + self.assertEqual(self.completer.attr_matches('egg.s'), + ['egg.{}('.format(x) for x in dir(str) + if x.startswith('s')]) def test_excessive_getattr(self): # Ensure getattr() is invoked no more than once per attribute -- cgit v0.12 From b4ce1fc31be5614d527d77c55018281ebbcd70ab Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 30 Nov 2015 03:18:29 +0000 Subject: Issue #5319: New Py_FinalizeEx() API to exit with status 120 on failure --- Doc/c-api/init.rst | 33 ++++++++++++++++++---------- Doc/c-api/intro.rst | 4 ++-- Doc/c-api/sys.rst | 14 +++++++----- Doc/extending/embedding.rst | 6 +++-- Doc/includes/run-func.c | 4 +++- Doc/library/sys.rst | 7 +++++- Doc/whatsnew/3.6.rst | 6 +++-- Include/pylifecycle.h | 1 + Lib/test/test_cmd_line.py | 3 ++- Misc/NEWS | 3 +++ Misc/SpecialBuilds.txt | 8 +++---- Modules/main.c | 8 +++++-- PC/bdist_wininst/install.c | 18 +++++++++------ PC/python3.def | 1 + Python/frozenmain.c | 4 +++- Python/pylifecycle.c | 52 +++++++++++++++++++++++++++++++------------- Python/pystate.c | 2 +- Tools/scripts/combinerefs.py | 4 ++-- 18 files changed, 120 insertions(+), 58 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 81823bf..465147c 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -25,7 +25,7 @@ Initializing and finalizing the interpreter triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() - single: Py_Finalize() + single: Py_FinalizeEx() Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; with the @@ -34,7 +34,7 @@ Initializing and finalizing the interpreter modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_Finalize` first). There is no return value; it is a + (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal error if the initialization fails. @@ -48,19 +48,20 @@ Initializing and finalizing the interpreter .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_Finalize` is called, this returns false until + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until :c:func:`Py_Initialize` is called again. -.. c:function:: void Py_Finalize() +.. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). There is no return - value; errors during finalization are ignored. + time (without calling :c:func:`Py_Initialize` again first). Normally the + return value is 0. If there were errors during finalization + (flushing buffered data), -1 is returned. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -79,7 +80,15 @@ Initializing and finalizing the interpreter freed. Some memory allocated by extension modules may not be freed. Some extensions may not work properly if their initialization routine is called more than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_Finalize` more than once. + :c:func:`Py_FinalizeEx` more than once. + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. Process-wide parameters @@ -107,7 +116,7 @@ Process-wide parameters Note that :data:`sys.stderr` always uses the "backslashreplace" error handler, regardless of this (or any other) setting. - If :c:func:`Py_Finalize` is called, this function will need to be called + If :c:func:`Py_FinalizeEx` is called, this function will need to be called again in order to affect subsequent calls to :c:func:`Py_Initialize`. Returns 0 if successful, a nonzero value on error (e.g. calling after the @@ -918,7 +927,7 @@ using the following functions: entry.) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: Py_Initialize() Extension modules are shared between (sub-)interpreters as follows: the first @@ -928,7 +937,7 @@ using the following functions: and filled with the contents of this copy; the extension's ``init`` function is not called. Note that this is different from what happens when an extension is imported after the interpreter has been completely re-initialized by calling - :c:func:`Py_Finalize` and :c:func:`Py_Initialize`; in that case, the extension's + :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that case, the extension's ``initmodule`` function *is* called again. .. index:: single: close() (in module os) @@ -936,14 +945,14 @@ using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_Finalize() + .. index:: single: Py_FinalizeEx() Destroy the (sub-)interpreter represented by the given thread state. The given thread state must be the current thread state. See the discussion of thread states below. When the call returns, the current thread state is *NULL*. All thread states associated with this interpreter are destroyed. (The global interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_Finalize` will destroy all sub-interpreters that + when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index bc3a752..95cbef5 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -578,9 +578,9 @@ Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to :c:func:`Py_Initialize`) or the application is simply done with its use of Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_Finalize`. The function :c:func:`Py_IsInitialized` returns +by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_Finalize` +these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` does *not* free all memory allocated by the Python interpreter, e.g. memory allocated by extension modules currently cannot be released. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 3d83b27..9ba6496 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -212,20 +212,24 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: exit() - Exit the current process. This calls :c:func:`Py_Finalize` and then calls the - standard C library function ``exit(status)``. + Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the + standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` + indicates an error, the exit status is set to 120. + + .. versionchanged:: 3.6 + Errors from finalization no longer ignored. .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_Finalize() + single: Py_FinalizeEx() single: cleanup functions - Register a cleanup function to be called by :c:func:`Py_Finalize`. The cleanup + Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup function will be called with no arguments and should return no value. At most 32 cleanup functions can be registered. When the registration is successful, :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index acd60ae..1546b1a 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -67,7 +67,9 @@ perform some operation on a file. :: Py_Initialize(); PyRun_SimpleString("from time import time,ctime\n" "print('Today is', ctime(time()))\n"); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + exit(120); + } PyMem_RawFree(program); return 0; } @@ -76,7 +78,7 @@ The :c:func:`Py_SetProgramName` function should be called before :c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time libraries. Next, the Python interpreter is initialized with :c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_Finalize` call shuts +that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts the interpreter down, followed by the end of the program. In a real program, you may want to get the Python script from another source, perhaps a text-editor routine, a file, or a database. Getting the Python code from a file can better diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c index 986d670..ead7bdd 100644 --- a/Doc/includes/run-func.c +++ b/Doc/includes/run-func.c @@ -63,6 +63,8 @@ main(int argc, char *argv[]) fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); return 1; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + return 120; + } return 0; } diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f6325cc..97b5899 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -255,7 +255,7 @@ always available. (defaulting to zero), or another type of object. If it is an integer, zero is considered "successful termination" and any nonzero value is considered "abnormal termination" by shells and the like. Most systems require it to be - in the range 0-127, and produce undefined results otherwise. Some systems + in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of @@ -268,6 +268,11 @@ always available. the process when called from the main thread, and the exception is not intercepted. + .. versionchanged:: 3.6 + If an error occurs in the cleanup after the Python interpreter + has caught :exc:`SystemExit` (such as an error flushing buffered data + in the standard streams), the exit status is changed to 120. + .. data:: flags diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index fc32fb5..f97c70f 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -171,7 +171,8 @@ Optimizations Build and C API Changes ======================= -* None yet. +* New :c:func:`Py_FinalizeEx` API which indicates if flushing buffered data + failed (:issue:`5319`). Deprecated @@ -247,4 +248,5 @@ Changes in the Python API Changes in the C API -------------------- -* None yet. +* :c:func:`Py_Exit` (and the main interpreter) now override the exit status + with 120 if flushing buffered data failed. See :issue:`5319`. diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index ccdebe2..e96eb70 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -27,6 +27,7 @@ PyAPI_FUNC(void) Py_InitializeEx(int); PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int); #endif PyAPI_FUNC(void) Py_Finalize(void); +PyAPI_FUNC(int) Py_FinalizeEx(void); PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 0feb63f..b410608 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -348,8 +348,9 @@ class CmdLineTest(unittest.TestCase): test.support.SuppressCrashReport().__enter__() sys.stdout.write('x') os.close(sys.stdout.fileno())""" - rc, out, err = assert_python_ok('-c', code) + rc, out, err = assert_python_failure('-c', code) self.assertEqual(b'', out) + self.assertEqual(120, rc) self.assertRegex(err.decode('ascii', 'ignore'), 'Exception ignored in.*\nOSError: .*') diff --git a/Misc/NEWS b/Misc/NEWS index 2cb06e9..ccbb3fb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: XXXX-XX-XX Core and Builtins ----------------- +- Issue #5319: New Py_FinalizeEx() API allowing Python to set an exit status + of 120 on failure to flush buffered streams. + - Issue #25485: telnetlib.Telnet is now a context manager. - Issue #24097: Fixed crash in object.__reduce__() if slot name is freed inside diff --git a/Misc/SpecialBuilds.txt b/Misc/SpecialBuilds.txt index 3004174..4b673fd 100644 --- a/Misc/SpecialBuilds.txt +++ b/Misc/SpecialBuilds.txt @@ -65,9 +65,9 @@ sys.getobjects(max[, type]) simply by virtue of being in the list. envvar PYTHONDUMPREFS - If this envvar exists, Py_Finalize() arranges to print a list of all + If this envvar exists, Py_FinalizeEx() arranges to print a list of all still-live heap objects. This is printed twice, in different formats, - before and after Py_Finalize has cleaned up everything it can clean up. The + before and after Py_FinalizeEx has cleaned up everything it can clean up. The first output block produces the repr() of each object so is more informative; however, a lot of stuff destined to die is still alive then. The second output block is much harder to work with (repr() can't be invoked @@ -144,7 +144,7 @@ Special gimmicks: envvar PYTHONMALLOCSTATS If this envvar exists, a report of pymalloc summary statistics is printed to - stderr whenever a new arena is allocated, and also by Py_Finalize(). + stderr whenever a new arena is allocated, and also by Py_FinalizeEx(). Changed in 2.5: The number of extra bytes allocated is 4*sizeof(size_t). Before it was 16 on all boxes, reflecting that Python couldn't make use of @@ -179,7 +179,7 @@ Each type object grows three new members: */ int tp_maxalloc; -Allocation and deallocation code keeps these counts up to date. Py_Finalize() +Allocation and deallocation code keeps these counts up to date. Py_FinalizeEx() displays a summary of the info returned by sys.getcounts() (see below), along with assorted other special allocation counts (like the number of tuple allocations satisfied by a tuple free-list, the number of 1-character strings diff --git a/Modules/main.c b/Modules/main.c index 2a9ea28..0fbdb69 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -654,7 +654,7 @@ Py_Main(int argc, wchar_t **argv) Py_SetProgramName(wbuf); /* Don't free wbuf, the argument to Py_SetProgramName - * must remain valid until the Py_Finalize is called. + * must remain valid until Py_FinalizeEx is called. */ } else { Py_SetProgramName(argv[0]); @@ -785,7 +785,11 @@ Py_Main(int argc, wchar_t **argv) sts = PyRun_AnyFileFlags(stdin, "", &cf) != 0; } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + /* Value unlikely to be confused with a non-error exit status or + other special meaning */ + sts = 120; + } #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c index f39b381..16eeb35 100644 --- a/PC/bdist_wininst/install.c +++ b/PC/bdist_wininst/install.c @@ -709,7 +709,7 @@ static int prepare_script_environment(HINSTANCE hPython) * 1 if the Python-dll does not export the functions we need * 2 if no install-script is specified in pathname * 3 if the install-script file could not be opened - * the return value of PyRun_SimpleString() otherwise, + * the return value of PyRun_SimpleString() or Py_FinalizeEx() otherwise, * which is 0 if everything is ok, -1 if an exception had occurred * in the install-script. */ @@ -722,7 +722,7 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, int, PySys_SetArgv, (int, wchar_t **)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, PyObject *, Py_BuildValue, (char *, ...)); DECLPROC(hPython, PyObject *, PyCFunction_New, (PyMethodDef *, PyObject *)); @@ -730,7 +730,7 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) DECLPROC(hPython, PyObject *, PyErr_Format, (PyObject *, char *)); if (!Py_Initialize || !PySys_SetArgv - || !PyRun_SimpleString || !Py_Finalize) + || !PyRun_SimpleString || !Py_FinalizeEx) return 1; if (!Py_BuildValue || !PyArg_ParseTuple || !PyErr_Format) @@ -777,7 +777,9 @@ do_run_installscript(HINSTANCE hPython, char *pathname, int argc, char **argv) } } } - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + result = -1; + } close(fh); return result; @@ -839,11 +841,11 @@ static int do_run_simple_script(HINSTANCE hPython, char *script) int rc; DECLPROC(hPython, void, Py_Initialize, (void)); DECLPROC(hPython, void, Py_SetProgramName, (wchar_t *)); - DECLPROC(hPython, void, Py_Finalize, (void)); + DECLPROC(hPython, int, Py_FinalizeEx, (void)); DECLPROC(hPython, int, PyRun_SimpleString, (char *)); DECLPROC(hPython, void, PyErr_Print, (void)); - if (!Py_Initialize || !Py_SetProgramName || !Py_Finalize || + if (!Py_Initialize || !Py_SetProgramName || !Py_FinalizeEx || !PyRun_SimpleString || !PyErr_Print) return -1; @@ -853,7 +855,9 @@ static int do_run_simple_script(HINSTANCE hPython, char *script) rc = PyRun_SimpleString(script); if (rc) PyErr_Print(); - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + rc = -1; + } return rc; } diff --git a/PC/python3.def b/PC/python3.def index e146e8f..e8d2d8c 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -648,6 +648,7 @@ EXPORTS Py_FatalError=python36.Py_FatalError Py_FileSystemDefaultEncoding=python36.Py_FileSystemDefaultEncoding DATA Py_Finalize=python36.Py_Finalize + Py_FinalizeEx=python36.Py_FinalizeEx Py_GetBuildInfo=python36.Py_GetBuildInfo Py_GetCompiler=python36.Py_GetCompiler Py_GetCopyright=python36.Py_GetCopyright diff --git a/Python/frozenmain.c b/Python/frozenmain.c index de8bd35..769b33d 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -99,7 +99,9 @@ Py_FrozenMain(int argc, char **argv) #ifdef MS_WINDOWS PyWinFreeze_ExeTerm(); #endif - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } error: PyMem_RawFree(argv_copy); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 857a543..b7f6ec8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -154,8 +154,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) return 0; } -/* Global initializations. Can be undone by Py_Finalize(). Don't - call this twice without an intervening Py_Finalize() call. When +/* Global initializations. Can be undone by Py_FinalizeEx(). Don't + call this twice without an intervening Py_FinalizeEx() call. When initializations fail, a fatal error is issued and the function does not return. On return, the first thread and interpreter state have been created. @@ -327,11 +327,11 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib) (void) PyThreadState_Swap(tstate); #ifdef WITH_THREAD - /* We can't call _PyEval_FiniThreads() in Py_Finalize because + /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because destroying the GIL might fail when it is being referenced from another running thread (see issue #9901). Instead we destroy the previously created GIL here, which ensures - that we can call Py_Initialize / Py_Finalize multiple times. */ + that we can call Py_Initialize / Py_FinalizeEx multiple times. */ _PyEval_FiniThreads(); /* Auto-thread-state API */ @@ -477,28 +477,35 @@ file_is_closed(PyObject *fobj) return r > 0; } -static void +static int flush_std_files(void) { PyObject *fout = _PySys_GetObjectId(&PyId_stdout); PyObject *ferr = _PySys_GetObjectId(&PyId_stderr); PyObject *tmp; + int status = 0; if (fout != NULL && fout != Py_None && !file_is_closed(fout)) { tmp = _PyObject_CallMethodId(fout, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_WriteUnraisable(fout); + status = -1; + } else Py_DECREF(tmp); } if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) { tmp = _PyObject_CallMethodId(ferr, &PyId_flush, ""); - if (tmp == NULL) + if (tmp == NULL) { PyErr_Clear(); + status = -1; + } else Py_DECREF(tmp); } + + return status; } /* Undo the effect of Py_Initialize(). @@ -515,14 +522,15 @@ flush_std_files(void) */ -void -Py_Finalize(void) +int +Py_FinalizeEx(void) { PyInterpreterState *interp; PyThreadState *tstate; + int status = 0; if (!initialized) - return; + return status; wait_for_thread_shutdown(); @@ -547,7 +555,9 @@ Py_Finalize(void) initialized = 0; /* Flush sys.stdout and sys.stderr */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Disable signal handling */ PyOS_FiniInterrupts(); @@ -576,7 +586,9 @@ Py_Finalize(void) PyImport_Cleanup(); /* Flush sys.stdout and sys.stderr (again, in case more was printed) */ - flush_std_files(); + if (flush_std_files() < 0) { + status = -1; + } /* Collect final garbage. This disposes of cycles created by * class definitions, for example. @@ -696,6 +708,13 @@ Py_Finalize(void) #endif call_ll_exitfuncs(); + return status; +} + +void +Py_Finalize(void) +{ + Py_FinalizeEx(); } /* Create and initialize a new interpreter and thread, and return the @@ -803,7 +822,7 @@ handle_error: frames, and that it is its interpreter's only remaining thread. It is a fatal error to violate these constraints. - (Py_Finalize() doesn't have these constraints -- it zaps + (Py_FinalizeEx() doesn't have these constraints -- it zaps everything, regardless.) Locking: as above. @@ -1016,7 +1035,8 @@ create_stdio(PyObject* io, mode = "rb"; buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi", fd, mode, buffering, - Py_None, Py_None, Py_None, 0); + Py_None, Py_None, /* encoding, errors */ + Py_None, 0); /* newline, closefd */ if (buf == NULL) goto error; @@ -1450,7 +1470,9 @@ call_ll_exitfuncs(void) void Py_Exit(int sts) { - Py_Finalize(); + if (Py_FinalizeEx() < 0) { + sts = 120; + } exit(sts); } diff --git a/Python/pystate.c b/Python/pystate.c index 7e0267a..50edfbb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -686,7 +686,7 @@ PyThreadState_IsCurrent(PyThreadState *tstate) } /* Internal initialization/finalization functions called by - Py_Initialize/Py_Finalize + Py_Initialize/Py_FinalizeEx */ void _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) diff --git a/Tools/scripts/combinerefs.py b/Tools/scripts/combinerefs.py index e10e49a..7ca9526 100755 --- a/Tools/scripts/combinerefs.py +++ b/Tools/scripts/combinerefs.py @@ -6,7 +6,7 @@ combinerefs path A helper for analyzing PYTHONDUMPREFS output. When the PYTHONDUMPREFS envar is set in a debug build, at Python shutdown -time Py_Finalize() prints the list of all live objects twice: first it +time Py_FinalizeEx() prints the list of all live objects twice: first it prints the repr() of each object while the interpreter is still fully intact. After cleaning up everything it can, it prints all remaining live objects again, but the second time just prints their addresses, refcounts, and type @@ -41,7 +41,7 @@ CAUTION: If object is a container type, it may not actually contain all the objects shown in the repr: the repr was captured from the first output block, and some of the containees may have been released since then. For example, it's common for the line showing the dict of interned strings to display -strings that no longer exist at the end of Py_Finalize; this can be recognized +strings that no longer exist at the end of Py_FinalizeEx; this can be recognized (albeit painfully) because such containees don't have a line of their own. The objects are listed in allocation order, with most-recently allocated -- cgit v0.12 From b28c3275c720b6908bebf9d545b90fc8789cf1fe Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Tue, 1 Dec 2015 19:59:53 +1100 Subject: Issue #25177: Fixed problem with the mean of very small and very large numbers. --- Lib/statistics.py | 185 +++++++++++++--------- Lib/test/test_statistics.py | 363 ++++++++++++++++++++++++++++++++++++++------ Misc/NEWS | 4 + 3 files changed, 433 insertions(+), 119 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index 9203cf1..518f546 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -104,6 +104,8 @@ import math from fractions import Fraction from decimal import Decimal +from itertools import groupby + # === Exceptions === @@ -115,86 +117,102 @@ class StatisticsError(ValueError): # === Private utilities === def _sum(data, start=0): - """_sum(data [, start]) -> value + """_sum(data [, start]) -> (type, sum, count) + + Return a high-precision sum of the given numeric data as a fraction, + together with the type to be converted to and the count of items. - Return a high-precision sum of the given numeric data. If optional - argument ``start`` is given, it is added to the total. If ``data`` is - empty, ``start`` (defaulting to 0) is returned. + If optional argument ``start`` is given, it is added to the total. + If ``data`` is empty, ``start`` (defaulting to 0) is returned. Examples -------- >>> _sum([3, 2.25, 4.5, -0.5, 1.0], 0.75) - 11.0 + (, Fraction(11, 1), 5) Some sources of round-off error will be avoided: >>> _sum([1e50, 1, -1e50] * 1000) # Built-in sum returns zero. - 1000.0 + (, Fraction(1000, 1), 3000) Fractions and Decimals are also supported: >>> from fractions import Fraction as F >>> _sum([F(2, 3), F(7, 5), F(1, 4), F(5, 6)]) - Fraction(63, 20) + (, Fraction(63, 20), 4) >>> from decimal import Decimal as D >>> data = [D("0.1375"), D("0.2108"), D("0.3061"), D("0.0419")] >>> _sum(data) - Decimal('0.6963') + (, Fraction(6963, 10000), 4) Mixed types are currently treated as an error, except that int is allowed. """ - # We fail as soon as we reach a value that is not an int or the type of - # the first value which is not an int. E.g. _sum([int, int, float, int]) - # is okay, but sum([int, int, float, Fraction]) is not. - allowed_types = {int, type(start)} + count = 0 n, d = _exact_ratio(start) - partials = {d: n} # map {denominator: sum of numerators} - # Micro-optimizations. - exact_ratio = _exact_ratio + partials = {d: n} partials_get = partials.get - # Add numerators for each denominator. - for x in data: - _check_type(type(x), allowed_types) - n, d = exact_ratio(x) - partials[d] = partials_get(d, 0) + n - # Find the expected result type. If allowed_types has only one item, it - # will be int; if it has two, use the one which isn't int. - assert len(allowed_types) in (1, 2) - if len(allowed_types) == 1: - assert allowed_types.pop() is int - T = int - else: - T = (allowed_types - {int}).pop() + T = _coerce(int, type(start)) + for typ, values in groupby(data, type): + T = _coerce(T, typ) # or raise TypeError + for n,d in map(_exact_ratio, values): + count += 1 + partials[d] = partials_get(d, 0) + n if None in partials: - assert issubclass(T, (float, Decimal)) - assert not math.isfinite(partials[None]) - return T(partials[None]) - total = Fraction() - for d, n in sorted(partials.items()): - total += Fraction(n, d) - if issubclass(T, int): - assert total.denominator == 1 - return T(total.numerator) - if issubclass(T, Decimal): - return T(total.numerator)/total.denominator - return T(total) - - -def _check_type(T, allowed): - if T not in allowed: - if len(allowed) == 1: - allowed.add(T) - else: - types = ', '.join([t.__name__ for t in allowed] + [T.__name__]) - raise TypeError("unsupported mixed types: %s" % types) + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + total = partials[None] + assert not _isfinite(total) + else: + # Sum all the partial sums using builtin sum. + # FIXME is this faster if we sum them in order of the denominator? + total = sum(Fraction(n, d) for d, n in sorted(partials.items())) + return (T, total, count) + + +def _isfinite(x): + try: + return x.is_finite() # Likely a Decimal. + except AttributeError: + return math.isfinite(x) # Coerces to float first. + + +def _coerce(T, S): + """Coerce types T and S to a common type, or raise TypeError. + + Coercion rules are currently an implementation detail. See the CoerceTest + test class in test_statistics for details. + """ + # See http://bugs.python.org/issue24068. + assert T is not bool, "initial type T is bool" + # If the types are the same, no need to coerce anything. Put this + # first, so that the usual case (no coercion needed) happens as soon + # as possible. + if T is S: return T + # Mixed int & other coerce to the other type. + if S is int or S is bool: return T + if T is int: return S + # If one is a (strict) subclass of the other, coerce to the subclass. + if issubclass(S, T): return S + if issubclass(T, S): return T + # Ints coerce to the other type. + if issubclass(T, int): return S + if issubclass(S, int): return T + # Mixed fraction & float coerces to float (or float subclass). + if issubclass(T, Fraction) and issubclass(S, float): + return S + if issubclass(T, float) and issubclass(S, Fraction): + return T + # Any other combination is disallowed. + msg = "don't know how to coerce %s and %s" + raise TypeError(msg % (T.__name__, S.__name__)) def _exact_ratio(x): - """Convert Real number x exactly to (numerator, denominator) pair. + """Return Real number x to exact (numerator, denominator) pair. >>> _exact_ratio(0.25) (1, 4) @@ -202,29 +220,31 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ try: + # Optimise the common case of floats. We expect that the most often + # used numeric type will be builtin floats, so try to make this as + # fast as possible. + if type(x) is float: + return x.as_integer_ratio() try: - # int, Fraction + # x may be an int, Fraction, or Integral ABC. return (x.numerator, x.denominator) except AttributeError: - # float try: + # x may be a float subclass. return x.as_integer_ratio() except AttributeError: - # Decimal try: + # x may be a Decimal. return _decimal_to_ratio(x) except AttributeError: - msg = "can't convert type '{}' to numerator/denominator" - raise TypeError(msg.format(type(x).__name__)) from None + # Just give up? + pass except (OverflowError, ValueError): - # INF or NAN - if __debug__: - # Decimal signalling NANs cannot be converted to float :-( - if isinstance(x, Decimal): - assert not x.is_finite() - else: - assert not math.isfinite(x) + # float NAN or INF. + assert not math.isfinite(x) return (x, None) + msg = "can't convert type '{}' to numerator/denominator" + raise TypeError(msg.format(type(x).__name__)) # FIXME This is faster than Fraction.from_decimal, but still too slow. @@ -239,7 +259,7 @@ def _decimal_to_ratio(d): sign, digits, exp = d.as_tuple() if exp in ('F', 'n', 'N'): # INF, NAN, sNAN assert not d.is_finite() - raise ValueError + return (d, None) num = 0 for digit in digits: num = num*10 + digit @@ -253,6 +273,24 @@ def _decimal_to_ratio(d): return (num, den) +def _convert(value, T): + """Convert value to given numeric type T.""" + if type(value) is T: + # This covers the cases where T is Fraction, or where value is + # a NAN or INF (Decimal or float). + return value + if issubclass(T, int) and value.denominator != 1: + T = float + try: + # FIXME: what do we do if this overflows? + return T(value) + except TypeError: + if issubclass(T, Decimal): + return T(value.numerator)/T(value.denominator) + else: + raise + + def _counts(data): # Generate a table of sorted (value, frequency) pairs. table = collections.Counter(iter(data)).most_common() @@ -290,7 +328,9 @@ def mean(data): n = len(data) if n < 1: raise StatisticsError('mean requires at least one data point') - return _sum(data)/n + T, total, count = _sum(data) + assert count == n + return _convert(total/n, T) # FIXME: investigate ways to calculate medians without sorting? Quickselect? @@ -460,12 +500,14 @@ def _ss(data, c=None): """ if c is None: c = mean(data) - ss = _sum((x-c)**2 for x in data) + T, total, count = _sum((x-c)**2 for x in data) # The following sum should mathematically equal zero, but due to rounding # error may not. - ss -= _sum((x-c) for x in data)**2/len(data) - assert not ss < 0, 'negative sum of square deviations: %f' % ss - return ss + U, total2, count2 = _sum((x-c) for x in data) + assert T == U and count == count2 + total -= total2**2/len(data) + assert not total < 0, 'negative sum of square deviations: %f' % total + return (T, total) def variance(data, xbar=None): @@ -511,8 +553,8 @@ def variance(data, xbar=None): n = len(data) if n < 2: raise StatisticsError('variance requires at least two data points') - ss = _ss(data, xbar) - return ss/(n-1) + T, ss = _ss(data, xbar) + return _convert(ss/(n-1), T) def pvariance(data, mu=None): @@ -560,7 +602,8 @@ def pvariance(data, mu=None): if n < 1: raise StatisticsError('pvariance requires at least one data point') ss = _ss(data, mu) - return ss/n + T, ss = _ss(data, mu) + return _convert(ss/n, T) def stdev(data, xbar=None): diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 758a481..0089ae8 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -21,6 +21,37 @@ import statistics # === Helper functions and class === +def _nan_equal(a, b): + """Return True if a and b are both the same kind of NAN. + + >>> _nan_equal(Decimal('NAN'), Decimal('NAN')) + True + >>> _nan_equal(Decimal('sNAN'), Decimal('sNAN')) + True + >>> _nan_equal(Decimal('NAN'), Decimal('sNAN')) + False + >>> _nan_equal(Decimal(42), Decimal('NAN')) + False + + >>> _nan_equal(float('NAN'), float('NAN')) + True + >>> _nan_equal(float('NAN'), 0.5) + False + + >>> _nan_equal(float('NAN'), Decimal('NAN')) + False + + NAN payloads are not compared. + """ + if type(a) is not type(b): + return False + if isinstance(a, float): + return math.isnan(a) and math.isnan(b) + aexp = a.as_tuple()[2] + bexp = b.as_tuple()[2] + return (aexp == bexp) and (aexp in ('n', 'N')) # Both NAN or both sNAN. + + def _calc_errors(actual, expected): """Return the absolute and relative errors between two numbers. @@ -675,15 +706,60 @@ class ExactRatioTest(unittest.TestCase): self.assertEqual(_exact_ratio(D("12.345")), (12345, 1000)) self.assertEqual(_exact_ratio(D("-1.98")), (-198, 100)) + def test_inf(self): + INF = float("INF") + class MyFloat(float): + pass + class MyDecimal(Decimal): + pass + for inf in (INF, -INF): + for type_ in (float, MyFloat, Decimal, MyDecimal): + x = type_(inf) + ratio = statistics._exact_ratio(x) + self.assertEqual(ratio, (x, None)) + self.assertEqual(type(ratio[0]), type_) + self.assertTrue(math.isinf(ratio[0])) + + def test_float_nan(self): + NAN = float("NAN") + class MyFloat(float): + pass + for nan in (NAN, MyFloat(NAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(math.isnan(ratio[0])) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + + def test_decimal_nan(self): + NAN = Decimal("NAN") + sNAN = Decimal("sNAN") + class MyDecimal(Decimal): + pass + for nan in (NAN, MyDecimal(NAN), sNAN, MyDecimal(sNAN)): + ratio = statistics._exact_ratio(nan) + self.assertTrue(_nan_equal(ratio[0], nan)) + self.assertIs(ratio[1], None) + self.assertEqual(type(ratio[0]), type(nan)) + class DecimalToRatioTest(unittest.TestCase): # Test _decimal_to_ratio private function. - def testSpecialsRaise(self): - # Test that NANs and INFs raise ValueError. - # Non-special values are covered by _exact_ratio above. - for d in (Decimal('NAN'), Decimal('sNAN'), Decimal('INF')): - self.assertRaises(ValueError, statistics._decimal_to_ratio, d) + def test_infinity(self): + # Test that INFs are handled correctly. + inf = Decimal('INF') + self.assertEqual(statistics._decimal_to_ratio(inf), (inf, None)) + self.assertEqual(statistics._decimal_to_ratio(-inf), (-inf, None)) + + def test_nan(self): + # Test that NANs are handled correctly. + for nan in (Decimal('NAN'), Decimal('sNAN')): + num, den = statistics._decimal_to_ratio(nan) + # Because NANs always compare non-equal, we cannot use assertEqual. + # Nor can we use an identity test, as we don't guarantee anything + # about the object identity. + self.assertTrue(_nan_equal(num, nan)) + self.assertIs(den, None) def test_sign(self): # Test sign is calculated correctly. @@ -718,25 +794,181 @@ class DecimalToRatioTest(unittest.TestCase): self.assertEqual(t, (147000, 1)) -class CheckTypeTest(unittest.TestCase): - # Test _check_type private function. +class IsFiniteTest(unittest.TestCase): + # Test _isfinite private function. + + def test_finite(self): + # Test that finite numbers are recognised as finite. + for x in (5, Fraction(1, 3), 2.5, Decimal("5.5")): + self.assertTrue(statistics._isfinite(x)) - def test_allowed(self): - # Test that a type which should be allowed is allowed. - allowed = set([int, float]) - statistics._check_type(int, allowed) - statistics._check_type(float, allowed) + def test_infinity(self): + # Test that INFs are not recognised as finite. + for x in (float("inf"), Decimal("inf")): + self.assertFalse(statistics._isfinite(x)) + + def test_nan(self): + # Test that NANs are not recognised as finite. + for x in (float("nan"), Decimal("NAN"), Decimal("sNAN")): + self.assertFalse(statistics._isfinite(x)) + + +class CoerceTest(unittest.TestCase): + # Test that private function _coerce correctly deals with types. + + # The coercion rules are currently an implementation detail, although at + # some point that should change. The tests and comments here define the + # correct implementation. + + # Pre-conditions of _coerce: + # + # - The first time _sum calls _coerce, the + # - coerce(T, S) will never be called with bool as the first argument; + # this is a pre-condition, guarded with an assertion. + + # + # - coerce(T, T) will always return T; we assume T is a valid numeric + # type. Violate this assumption at your own risk. + # + # - Apart from as above, bool is treated as if it were actually int. + # + # - coerce(int, X) and coerce(X, int) return X. + # - + def test_bool(self): + # bool is somewhat special, due to the pre-condition that it is + # never given as the first argument to _coerce, and that it cannot + # be subclassed. So we test it specially. + for T in (int, float, Fraction, Decimal): + self.assertIs(statistics._coerce(T, bool), T) + class MyClass(T): pass + self.assertIs(statistics._coerce(MyClass, bool), MyClass) + + def assertCoerceTo(self, A, B): + """Assert that type A coerces to B.""" + self.assertIs(statistics._coerce(A, B), B) + self.assertIs(statistics._coerce(B, A), B) + + def check_coerce_to(self, A, B): + """Checks that type A coerces to B, including subclasses.""" + # Assert that type A is coerced to B. + self.assertCoerceTo(A, B) + # Subclasses of A are also coerced to B. + class SubclassOfA(A): pass + self.assertCoerceTo(SubclassOfA, B) + # A, and subclasses of A, are coerced to subclasses of B. + class SubclassOfB(B): pass + self.assertCoerceTo(A, SubclassOfB) + self.assertCoerceTo(SubclassOfA, SubclassOfB) + + def assertCoerceRaises(self, A, B): + """Assert that coercing A to B, or vice versa, raises TypeError.""" + self.assertRaises(TypeError, statistics._coerce, (A, B)) + self.assertRaises(TypeError, statistics._coerce, (B, A)) + + def check_type_coercions(self, T): + """Check that type T coerces correctly with subclasses of itself.""" + assert T is not bool + # Coercing a type with itself returns the same type. + self.assertIs(statistics._coerce(T, T), T) + # Coercing a type with a subclass of itself returns the subclass. + class U(T): pass + class V(T): pass + class W(U): pass + for typ in (U, V, W): + self.assertCoerceTo(T, typ) + self.assertCoerceTo(U, W) + # Coercing two subclasses that aren't parent/child is an error. + self.assertCoerceRaises(U, V) + self.assertCoerceRaises(V, W) - def test_not_allowed(self): - # Test that a type which should not be allowed raises. - allowed = set([int, float]) - self.assertRaises(TypeError, statistics._check_type, Decimal, allowed) + def test_int(self): + # Check that int coerces correctly. + self.check_type_coercions(int) + for typ in (float, Fraction, Decimal): + self.check_coerce_to(int, typ) - def test_add_to_allowed(self): - # Test that a second type will be added to the allowed set. - allowed = set([int]) - statistics._check_type(float, allowed) - self.assertEqual(allowed, set([int, float])) + def test_fraction(self): + # Check that Fraction coerces correctly. + self.check_type_coercions(Fraction) + self.check_coerce_to(Fraction, float) + + def test_decimal(self): + # Check that Decimal coerces correctly. + self.check_type_coercions(Decimal) + + def test_float(self): + # Check that float coerces correctly. + self.check_type_coercions(float) + + def test_non_numeric_types(self): + for bad_type in (str, list, type(None), tuple, dict): + for good_type in (int, float, Fraction, Decimal): + self.assertCoerceRaises(good_type, bad_type) + + def test_incompatible_types(self): + # Test that incompatible types raise. + for T in (float, Fraction): + class MySubclass(T): pass + self.assertCoerceRaises(T, Decimal) + self.assertCoerceRaises(MySubclass, Decimal) + + +class ConvertTest(unittest.TestCase): + # Test private _convert function. + + def check_exact_equal(self, x, y): + """Check that x equals y, and has the same type as well.""" + self.assertEqual(x, y) + self.assertIs(type(x), type(y)) + + def test_int(self): + # Test conversions to int. + x = statistics._convert(Fraction(71), int) + self.check_exact_equal(x, 71) + class MyInt(int): pass + x = statistics._convert(Fraction(17), MyInt) + self.check_exact_equal(x, MyInt(17)) + + def test_fraction(self): + # Test conversions to Fraction. + x = statistics._convert(Fraction(95, 99), Fraction) + self.check_exact_equal(x, Fraction(95, 99)) + class MyFraction(Fraction): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(71, 13), MyFraction) + self.check_exact_equal(x, MyFraction(71, 13)) + + def test_float(self): + # Test conversions to float. + x = statistics._convert(Fraction(-1, 2), float) + self.check_exact_equal(x, -0.5) + class MyFloat(float): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(9, 8), MyFloat) + self.check_exact_equal(x, MyFloat(1.125)) + + def test_decimal(self): + # Test conversions to Decimal. + x = statistics._convert(Fraction(1, 40), Decimal) + self.check_exact_equal(x, Decimal("0.025")) + class MyDecimal(Decimal): + def __truediv__(self, other): + return self.__class__(super().__truediv__(other)) + x = statistics._convert(Fraction(-15, 16), MyDecimal) + self.check_exact_equal(x, MyDecimal("-0.9375")) + + def test_inf(self): + for INF in (float('inf'), Decimal('inf')): + for inf in (INF, -INF): + x = statistics._convert(inf, type(inf)) + self.check_exact_equal(x, inf) + + def test_nan(self): + for nan in (float('nan'), Decimal('NAN'), Decimal('sNAN')): + x = statistics._convert(nan, type(nan)) + self.assertTrue(_nan_equal(x, nan)) # === Tests for public functions === @@ -874,52 +1106,71 @@ class UnivariateTypeMixin: self.assertIs(type(result), kind) -class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): +class TestSumCommon(UnivariateCommonMixin, UnivariateTypeMixin): + # Common test cases for statistics._sum() function. + + # This test suite looks only at the numeric value returned by _sum, + # after conversion to the appropriate type. + def setUp(self): + def simplified_sum(*args): + T, value, n = statistics._sum(*args) + return statistics._coerce(value, T) + self.func = simplified_sum + + +class TestSum(NumericTestCase): # Test cases for statistics._sum() function. + # These tests look at the entire three value tuple returned by _sum. + def setUp(self): self.func = statistics._sum def test_empty_data(self): # Override test for empty data. for data in ([], (), iter([])): - self.assertEqual(self.func(data), 0) - self.assertEqual(self.func(data, 23), 23) - self.assertEqual(self.func(data, 2.3), 2.3) + self.assertEqual(self.func(data), (int, Fraction(0), 0)) + self.assertEqual(self.func(data, 23), (int, Fraction(23), 0)) + self.assertEqual(self.func(data, 2.3), (float, Fraction(2.3), 0)) def test_ints(self): - self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), 60) - self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), 1008) + self.assertEqual(self.func([1, 5, 3, -4, -8, 20, 42, 1]), + (int, Fraction(60), 8)) + self.assertEqual(self.func([4, 2, 3, -8, 7], 1000), + (int, Fraction(1008), 5)) def test_floats(self): - self.assertEqual(self.func([0.25]*20), 5.0) - self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), 3.125) + self.assertEqual(self.func([0.25]*20), + (float, Fraction(5.0), 20)) + self.assertEqual(self.func([0.125, 0.25, 0.5, 0.75], 1.5), + (float, Fraction(3.125), 4)) def test_fractions(self): - F = Fraction - self.assertEqual(self.func([Fraction(1, 1000)]*500), Fraction(1, 2)) + self.assertEqual(self.func([Fraction(1, 1000)]*500), + (Fraction, Fraction(1, 2), 500)) def test_decimals(self): D = Decimal data = [D("0.001"), D("5.246"), D("1.702"), D("-0.025"), D("3.974"), D("2.328"), D("4.617"), D("2.843"), ] - self.assertEqual(self.func(data), Decimal("20.686")) + self.assertEqual(self.func(data), + (Decimal, Decimal("20.686"), 8)) def test_compare_with_math_fsum(self): # Compare with the math.fsum function. # Ideally we ought to get the exact same result, but sometimes # we differ by a very slight amount :-( data = [random.uniform(-100, 1000) for _ in range(1000)] - self.assertApproxEqual(self.func(data), math.fsum(data), rel=2e-16) + self.assertApproxEqual(float(self.func(data)[1]), math.fsum(data), rel=2e-16) def test_start_argument(self): # Test that the optional start argument works correctly. data = [random.uniform(1, 1000) for _ in range(100)] - t = self.func(data) - self.assertEqual(t+42, self.func(data, 42)) - self.assertEqual(t-23, self.func(data, -23)) - self.assertEqual(t+1e20, self.func(data, 1e20)) + t = self.func(data)[1] + self.assertEqual(t+42, self.func(data, 42)[1]) + self.assertEqual(t-23, self.func(data, -23)[1]) + self.assertEqual(t+Fraction(1e20), self.func(data, 1e20)[1]) def test_strings_fail(self): # Sum of strings should fail. @@ -934,7 +1185,7 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): def test_mixed_sum(self): # Mixed input types are not (currently) allowed. # Check that mixed data types fail. - self.assertRaises(TypeError, self.func, [1, 2.0, Fraction(1, 2)]) + self.assertRaises(TypeError, self.func, [1, 2.0, Decimal(1)]) # And so does mixed start argument. self.assertRaises(TypeError, self.func, [1, 2.0], Decimal(1)) @@ -942,11 +1193,14 @@ class TestSum(NumericTestCase, UnivariateCommonMixin, UnivariateTypeMixin): class SumTortureTest(NumericTestCase): def test_torture(self): # Tim Peters' torture test for sum, and variants of same. - self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), 20000.0) - self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), 20000.0) - self.assertApproxEqual( - statistics._sum([1e-100, 1, 1e-100, -1]*10000), 2.0e-96, rel=5e-16 - ) + self.assertEqual(statistics._sum([1, 1e100, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + self.assertEqual(statistics._sum([1e100, 1, 1, -1e100]*10000), + (float, Fraction(20000.0), 40000)) + T, num, count = statistics._sum([1e-100, 1, 1e-100, -1]*10000) + self.assertIs(T, float) + self.assertEqual(count, 40000) + self.assertApproxEqual(float(num), 2.0e-96, rel=5e-16) class SumSpecialValues(NumericTestCase): @@ -955,7 +1209,7 @@ class SumSpecialValues(NumericTestCase): def test_nan(self): for type_ in (float, Decimal): nan = type_('nan') - result = statistics._sum([1, nan, 2]) + result = statistics._sum([1, nan, 2])[1] self.assertIs(type(result), type_) self.assertTrue(math.isnan(result)) @@ -968,10 +1222,10 @@ class SumSpecialValues(NumericTestCase): def do_test_inf(self, inf): # Adding a single infinity gives infinity. - result = statistics._sum([1, 2, inf, 3]) + result = statistics._sum([1, 2, inf, 3])[1] self.check_infinity(result, inf) # Adding two infinities of the same sign also gives infinity. - result = statistics._sum([1, 2, inf, 3, inf, 4]) + result = statistics._sum([1, 2, inf, 3, inf, 4])[1] self.check_infinity(result, inf) def test_float_inf(self): @@ -987,7 +1241,7 @@ class SumSpecialValues(NumericTestCase): def test_float_mismatched_infs(self): # Test that adding two infinities of opposite sign gives a NAN. inf = float('inf') - result = statistics._sum([1, 2, inf, 3, -inf, 4]) + result = statistics._sum([1, 2, inf, 3, -inf, 4])[1] self.assertTrue(math.isnan(result)) def test_decimal_extendedcontext_mismatched_infs_to_nan(self): @@ -995,7 +1249,7 @@ class SumSpecialValues(NumericTestCase): inf = Decimal('inf') data = [1, 2, inf, 3, -inf, 4] with decimal.localcontext(decimal.ExtendedContext): - self.assertTrue(math.isnan(statistics._sum(data))) + self.assertTrue(math.isnan(statistics._sum(data)[1])) def test_decimal_basiccontext_mismatched_infs_to_nan(self): # Test adding Decimal INFs with opposite sign raises InvalidOperation. @@ -1111,6 +1365,19 @@ class TestMean(NumericTestCase, AverageMixin, UnivariateTypeMixin): d = Decimal('1e4') self.assertEqual(statistics.mean([d]), d) + def test_regression_25177(self): + # Regression test for issue 25177. + # Ensure very big and very small floats don't overflow. + # See http://bugs.python.org/issue25177. + self.assertEqual(statistics.mean( + [8.988465674311579e+307, 8.98846567431158e+307]), + 8.98846567431158e+307) + big = 8.98846567431158e+307 + tiny = 5e-324 + for n in (2, 3, 5, 200): + self.assertEqual(statistics.mean([big]*n), big) + self.assertEqual(statistics.mean([tiny]*n), tiny) + class TestMedian(NumericTestCase, AverageMixin): # Common tests for median and all median.* functions. diff --git a/Misc/NEWS b/Misc/NEWS index 649c891..fdefafc 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -107,6 +107,10 @@ Core and Builtins Library ------- +- Issue #25177: Fixed problem with the mean of very small and very large + numbers. As a side effect, statistics.mean and statistics.variance should + be significantly faster. + - Issue #25718: Fixed copying object with state with boolean value is false. - Issue #10131: Fixed deep copying of minidom documents. Based on patch -- cgit v0.12 From 65ca88e4e0b3d26e625d1aecd7c0fe0beb5e3b1d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 Dec 2015 15:19:42 -0800 Subject: Issue #25771: Tweak ValueError message when package isn't specified for importlib.util.resolve_name() but is needed. Thanks to Martin Panter for the bug report. --- Lib/importlib/util.py | 4 ++-- Misc/NEWS | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 1dbff26..39cb0f7 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -22,8 +22,8 @@ def resolve_name(name, package): if not name.startswith('.'): return name elif not package: - raise ValueError('{!r} is not a relative name ' - '(no leading dot)'.format(name)) + raise ValueError(f'no package specified for {repr(name)} ' + '(required for relative module names)') level = 0 for character in name: if character != '.': diff --git a/Misc/NEWS b/Misc/NEWS index f87ed38..f4e7e54 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,9 @@ Core and Builtins Library ------- +- Issue #25771: Tweak the exception message for importlib.util.resolve_name() + when 'package' isn't specified but necessary. + - Issue #6478: _strptime's regexp cache now is reset after changing timezone with time.tzset(). -- cgit v0.12 From da0f2a1f5250a36c6067e035dcc9064b00b20bf5 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 04:16:45 +0000 Subject: Issue #25764: Attempt to debug and skip OS X setrlimit() failure --- Lib/test/test_subprocess.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 0448d64..b68def5 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1516,10 +1516,16 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC + from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + try: + setrlimit(RLIMIT_NPROC, limits) + except ValueError as err: + # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots + print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") + self.skipTest("Setting existing NPROC limit failed") [_, hard] = limits setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) -- cgit v0.12 From aa0780801c7a53760296de5c7c9ee0fe6562b6bc Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 05:42:18 +0000 Subject: Issue #25764: OS X now failing on the second setrlimit() call --- Lib/test/test_subprocess.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index b68def5..ed1b213 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1520,14 +1520,17 @@ class POSIXProcessTestCase(BaseTestCase): except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) + [_, hard] = limits try: setrlimit(RLIMIT_NPROC, limits) + setrlimit(RLIMIT_NPROC, (0, hard)) except ValueError as err: - # Seems to happen on AMD64 Snow Leop and x86-64 Yosemite buildbots - print(f"Setting NPROC to {limits!r}: {err!r}, RLIM_INFINITY={RLIM_INFINITY!r}") - self.skipTest("Setting existing NPROC limit failed") - [_, hard] = limits - setrlimit(RLIMIT_NPROC, (0, hard)) + # Seems to happen on various OS X buildbots + print( + f"Setting NPROC failed: {err!r}, limits={limits!r}, " + f"RLIM_INFINITY={RLIM_INFINITY!r}, " + f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") + self.skipTest("Setting NPROC limit failed") self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v0.12 From 0d559cf72a5c80e7deda214022deb56ec8db1707 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 5 Dec 2015 10:18:25 +0000 Subject: Issue #25764: Remove test debugging --- Lib/test/test_subprocess.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 81c2672..b32ef97 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1518,21 +1518,12 @@ class POSIXProcessTestCase(BaseTestCase): # The internal code did not preserve the previous exception when # re-enabling garbage collection try: - from resource import getrlimit, setrlimit, RLIMIT_NPROC, RLIM_INFINITY + from resource import getrlimit, setrlimit, RLIMIT_NPROC except ImportError as err: self.skipTest(err) # RLIMIT_NPROC is specific to Linux and BSD limits = getrlimit(RLIMIT_NPROC) [_, hard] = limits - try: - setrlimit(RLIMIT_NPROC, limits) - setrlimit(RLIMIT_NPROC, (0, hard)) - except ValueError as err: - # Seems to happen on various OS X buildbots - print( - f"Setting NPROC failed: {err!r}, limits={limits!r}, " - f"RLIM_INFINITY={RLIM_INFINITY!r}, " - f"getrlimit() -> {getrlimit(RLIMIT_NPROC)!r}") - self.skipTest("Setting NPROC limit failed") + setrlimit(RLIMIT_NPROC, (0, hard)) self.addCleanup(setrlimit, RLIMIT_NPROC, limits) # Forking should raise EAGAIN, translated to BlockingIOError with self.assertRaises(BlockingIOError): -- cgit v0.12 From 59fb6342a4ffdc23e1269a418734f4fc0f984873 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Dec 2015 22:01:35 +0200 Subject: Issue #25761: Improved detecting errors in broken pickle data. --- Lib/pickle.py | 88 +++++++++++++++---------------------- Lib/test/pickletester.py | 17 +++----- Lib/test/test_pickle.py | 5 --- Misc/NEWS | 2 + Modules/_pickle.c | 111 +++++++++++++++++++++++++++++------------------ 5 files changed, 113 insertions(+), 110 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index 53978fb..a60b1b7 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1031,7 +1031,7 @@ class _Unpickler: self._unframer = _Unframer(self._file_read, self._file_readline) self.read = self._unframer.read self.readline = self._unframer.readline - self.mark = object() # any new unique object + self.metastack = [] self.stack = [] self.append = self.stack.append self.proto = 0 @@ -1047,20 +1047,12 @@ class _Unpickler: except _Stop as stopinst: return stopinst.value - # Return largest index k such that self.stack[k] is self.mark. - # If the stack doesn't contain a mark, eventually raises IndexError. - # This could be sped by maintaining another stack, of indices at which - # the mark appears. For that matter, the latter stack would suffice, - # and we wouldn't need to push mark objects on self.stack at all. - # Doing so is probably a good thing, though, since if the pickle is - # corrupt (or hostile) we may get a clue from finding self.mark embedded - # in unpickled objects. - def marker(self): - stack = self.stack - mark = self.mark - k = len(stack)-1 - while stack[k] is not mark: k = k-1 - return k + # Return a list of items pushed in the stack after last MARK instruction. + def pop_mark(self): + items = self.stack + self.stack = self.metastack.pop() + self.append = self.stack.append + return items def persistent_load(self, pid): raise UnpicklingError("unsupported persistent id encountered") @@ -1237,8 +1229,8 @@ class _Unpickler: dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode def load_tuple(self): - k = self.marker() - self.stack[k:] = [tuple(self.stack[k+1:])] + items = self.pop_mark() + self.append(tuple(items)) dispatch[TUPLE[0]] = load_tuple def load_empty_tuple(self): @@ -1270,21 +1262,20 @@ class _Unpickler: dispatch[EMPTY_SET[0]] = load_empty_set def load_frozenset(self): - k = self.marker() - self.stack[k:] = [frozenset(self.stack[k+1:])] + items = self.pop_mark() + self.append(frozenset(items)) dispatch[FROZENSET[0]] = load_frozenset def load_list(self): - k = self.marker() - self.stack[k:] = [self.stack[k+1:]] + items = self.pop_mark() + self.append(items) dispatch[LIST[0]] = load_list def load_dict(self): - k = self.marker() - items = self.stack[k+1:] + items = self.pop_mark() d = {items[i]: items[i+1] for i in range(0, len(items), 2)} - self.stack[k:] = [d] + self.append(d) dispatch[DICT[0]] = load_dict # INST and OBJ differ only in how they get a class object. It's not @@ -1292,9 +1283,7 @@ class _Unpickler: # previously diverged and grew different bugs. # klass is the class to instantiate, and k points to the topmost mark # object, following which are the arguments for klass.__init__. - def _instantiate(self, klass, k): - args = tuple(self.stack[k+1:]) - del self.stack[k:] + def _instantiate(self, klass, args): if (args or not isinstance(klass, type) or hasattr(klass, "__getinitargs__")): try: @@ -1310,14 +1299,14 @@ class _Unpickler: module = self.readline()[:-1].decode("ascii") name = self.readline()[:-1].decode("ascii") klass = self.find_class(module, name) - self._instantiate(klass, self.marker()) + self._instantiate(klass, self.pop_mark()) dispatch[INST[0]] = load_inst def load_obj(self): # Stack is ... markobject classobject arg1 arg2 ... - k = self.marker() - klass = self.stack.pop(k+1) - self._instantiate(klass, k) + args = self.pop_mark() + cls = args.pop(0) + self._instantiate(cls, args) dispatch[OBJ[0]] = load_obj def load_newobj(self): @@ -1402,12 +1391,14 @@ class _Unpickler: dispatch[REDUCE[0]] = load_reduce def load_pop(self): - del self.stack[-1] + if self.stack: + del self.stack[-1] + else: + self.pop_mark() dispatch[POP[0]] = load_pop def load_pop_mark(self): - k = self.marker() - del self.stack[k:] + self.pop_mark() dispatch[POP_MARK[0]] = load_pop_mark def load_dup(self): @@ -1463,17 +1454,14 @@ class _Unpickler: dispatch[APPEND[0]] = load_append def load_appends(self): - stack = self.stack - mark = self.marker() - list_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + list_obj = self.stack[-1] if isinstance(list_obj, list): list_obj.extend(items) else: append = list_obj.append for item in items: append(item) - del stack[mark:] dispatch[APPENDS[0]] = load_appends def load_setitem(self): @@ -1485,27 +1473,21 @@ class _Unpickler: dispatch[SETITEM[0]] = load_setitem def load_setitems(self): - stack = self.stack - mark = self.marker() - dict = stack[mark - 1] - for i in range(mark + 1, len(stack), 2): - dict[stack[i]] = stack[i + 1] - - del stack[mark:] + items = self.pop_mark() + dict = self.stack[-1] + for i in range(0, len(items), 2): + dict[items[i]] = items[i + 1] dispatch[SETITEMS[0]] = load_setitems def load_additems(self): - stack = self.stack - mark = self.marker() - set_obj = stack[mark - 1] - items = stack[mark + 1:] + items = self.pop_mark() + set_obj = self.stack[-1] if isinstance(set_obj, set): set_obj.update(items) else: add = set_obj.add for item in items: add(item) - del stack[mark:] dispatch[ADDITEMS[0]] = load_additems def load_build(self): @@ -1533,7 +1515,9 @@ class _Unpickler: dispatch[BUILD[0]] = load_build def load_mark(self): - self.append(self.mark) + self.metastack.append(self.stack) + self.stack = [] + self.append = self.stack.append dispatch[MARK[0]] = load_mark def load_stop(self): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 608c35a..217aa3d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1000,7 +1000,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'0', # POP b'1', # POP_MARK b'2', # DUP - # b'(2', # PyUnpickler doesn't raise + b'(2', b'R', # REDUCE b')R', b'a', # APPEND @@ -1009,7 +1009,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'Nb', b'd', # DICT b'e', # APPENDS - # b'(e', # PyUnpickler raises AttributeError + b'(e', b'ibuiltins\nlist\n', # INST b'l', # LIST b'o', # OBJ @@ -1022,7 +1022,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'NNs', b't', # TUPLE b'u', # SETITEMS - # b'(u', # PyUnpickler doesn't raise + b'(u', b'}(Nu', b'\x81', # NEWOBJ b')\x81', @@ -1033,7 +1033,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N\x87', b'NN\x87', b'\x90', # ADDITEMS - # b'(\x90', # PyUnpickler raises AttributeError + b'(\x90', b'\x91', # FROZENSET b'\x92', # NEWOBJ_EX b')}\x92', @@ -1046,7 +1046,7 @@ class AbstractUnpickleTests(unittest.TestCase): def test_bad_mark(self): badpickles = [ - # b'N(.', # STOP + b'N(.', # STOP b'N(2', # DUP b'cbuiltins\nlist\n)(R', # REDUCE b'cbuiltins\nlist\n()R', @@ -1081,7 +1081,7 @@ class AbstractUnpickleTests(unittest.TestCase): b'N(\x94', # MEMOIZE ] for p in badpickles: - self.check_unpickling_error(self.bad_mark_errors, p) + self.check_unpickling_error(self.bad_stack_errors, p) def test_truncated_data(self): self.check_unpickling_error(EOFError, b'') @@ -2581,11 +2581,6 @@ class AbstractPickleModuleTests(unittest.TestCase): self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) - def test_bad_input(self): - # Test issue4298 - s = bytes([0x58, 0, 0, 0, 0x54]) - self.assertRaises(EOFError, pickle.loads, s) - class AbstractPersistentPicklerTests(unittest.TestCase): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index bd38cfb..6b97315 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -33,8 +33,6 @@ class PyUnpicklerTests(AbstractUnpickleTests): unpickler = pickle._Unpickler bad_stack_errors = (IndexError,) - bad_mark_errors = (IndexError, pickle.UnpicklingError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -69,8 +67,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, pickler = pickle._Pickler unpickler = pickle._Unpickler bad_stack_errors = (pickle.UnpicklingError, IndexError) - bad_mark_errors = (pickle.UnpicklingError, IndexError, - TypeError, AttributeError, EOFError) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError, struct.error, IndexError, ImportError) @@ -132,7 +128,6 @@ if has_c_implementation: class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) - bad_mark_errors = (EOFError,) truncated_errors = (pickle.UnpicklingError, EOFError, AttributeError, ValueError) diff --git a/Misc/NEWS b/Misc/NEWS index 026a19a..578d2a3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,8 @@ Core and Builtins Library ------- +- Issue #25761: Improved detecting errors in broken pickle data. + - Issue #25717: Restore the previous behaviour of tolerating most fstat() errors when opening files. This was a regression in 3.5a1, and stopped anonymous temporary files from working in special cases. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 3ddf6a0..38598c5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -370,18 +370,12 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ -static int -stack_underflow(void) -{ - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "unpickling stack underflow"); - return -1; -} - /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD PyObject **data; + int mark_set; /* is MARK set? */ + Py_ssize_t fence; /* position of top MARK or 0 */ Py_ssize_t allocated; /* number of slots in data allocated */ } Pdata; @@ -412,6 +406,8 @@ Pdata_New(void) if (!(self = PyObject_New(Pdata, &Pdata_Type))) return NULL; Py_SIZE(self) = 0; + self->mark_set = 0; + self->fence = 0; self->allocated = 8; self->data = PyMem_MALLOC(self->allocated * sizeof(PyObject *)); if (self->data) @@ -429,8 +425,7 @@ Pdata_clear(Pdata *self, Py_ssize_t clearto) { Py_ssize_t i = Py_SIZE(self); - if (clearto < 0) - return stack_underflow(); + assert(clearto >= self->fence); if (clearto >= i) return 0; @@ -466,6 +461,17 @@ Pdata_grow(Pdata *self) return -1; } +static int +Pdata_stack_underflow(Pdata *self) +{ + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, + self->mark_set ? + "unexpected MARK found" : + "unpickling stack underflow"); + return -1; +} + /* D is a Pdata*. Pop the topmost element and store it into V, which * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError * is raised and V is set to NULL. @@ -473,9 +479,8 @@ Pdata_grow(Pdata *self) static PyObject * Pdata_pop(Pdata *self) { - if (Py_SIZE(self) == 0) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "bad pickle data"); + if (Py_SIZE(self) <= self->fence) { + Pdata_stack_underflow(self); return NULL; } return self->data[--Py_SIZE(self)]; @@ -507,6 +512,10 @@ Pdata_poptuple(Pdata *self, Py_ssize_t start) PyObject *tuple; Py_ssize_t len, i, j; + if (start < self->fence) { + Pdata_stack_underflow(self); + return NULL; + } len = Py_SIZE(self) - start; tuple = PyTuple_New(len); if (tuple == NULL) @@ -4585,13 +4594,19 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) static Py_ssize_t marker(UnpicklerObject *self) { - PickleState *st = _Pickle_GetGlobalState(); + Py_ssize_t mark; + if (self->num_marks < 1) { + PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "could not find MARK"); return -1; } - return self->marks[--self->num_marks]; + mark = self->marks[--self->num_marks]; + self->stack->mark_set = self->num_marks != 0; + self->stack->fence = self->num_marks ? + self->marks[self->num_marks - 1] : 0; + return mark; } static int @@ -5052,7 +5067,7 @@ load_counted_tuple(UnpicklerObject *self, int len) PyObject *tuple; if (Py_SIZE(self->stack) < len) - return stack_underflow(); + return Pdata_stack_underflow(self->stack); tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len); if (tuple == NULL) @@ -5134,6 +5149,12 @@ load_dict(UnpicklerObject *self) if ((dict = PyDict_New()) == NULL) return -1; + if ((j - i) % 2 != 0) { + PickleState *st = _Pickle_GetGlobalState(); + PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); + return -1; + } + for (k = i + 1; k < j; k += 2) { key = self->stack->data[k - 1]; value = self->stack->data[k]; @@ -5201,7 +5222,7 @@ load_obj(UnpicklerObject *self) return -1; if (Py_SIZE(self->stack) - i < 1) - return stack_underflow(); + return Pdata_stack_underflow(self->stack); args = Pdata_poptuple(self->stack, i + 1); if (args == NULL) @@ -5518,12 +5539,15 @@ load_pop(UnpicklerObject *self) */ if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) { self->num_marks--; - } else if (len > 0) { + self->stack->mark_set = self->num_marks != 0; + self->stack->fence = self->num_marks ? + self->marks[self->num_marks - 1] : 0; + } else if (len <= self->stack->fence) + return Pdata_stack_underflow(self->stack); + else { len--; Py_DECREF(self->stack->data[len]); Py_SIZE(self->stack) = len; - } else { - return stack_underflow(); } return 0; } @@ -5545,10 +5569,10 @@ static int load_dup(UnpicklerObject *self) { PyObject *last; - Py_ssize_t len; + Py_ssize_t len = Py_SIZE(self->stack); - if ((len = Py_SIZE(self->stack)) <= 0) - return stack_underflow(); + if (len <= self->stack->fence) + return Pdata_stack_underflow(self->stack); last = self->stack->data[len - 1]; PDATA_APPEND(self->stack, last, -1); return 0; @@ -5731,8 +5755,8 @@ load_put(UnpicklerObject *self) return -1; if (len < 2) return bad_readline(); - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; key = PyLong_FromString(s, NULL, 10); @@ -5760,8 +5784,8 @@ load_binput(UnpicklerObject *self) if (_Unpickler_Read(self, &s, 1) < 0) return -1; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = Py_CHARMASK(s[0]); @@ -5779,8 +5803,8 @@ load_long_binput(UnpicklerObject *self) if (_Unpickler_Read(self, &s, 4) < 0) return -1; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = calc_binsize(s, 4); @@ -5798,8 +5822,8 @@ load_memoize(UnpicklerObject *self) { PyObject *value; - if (Py_SIZE(self->stack) <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) <= self->stack->fence) + return Pdata_stack_underflow(self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; return _Unpickler_MemoPut(self, self->memo_len, value); @@ -5813,8 +5837,8 @@ do_append(UnpicklerObject *self, Py_ssize_t x) Py_ssize_t len, i; len = Py_SIZE(self->stack); - if (x > len || x <= 0) - return stack_underflow(); + if (x > len || x <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == x) /* nothing to do */ return 0; @@ -5863,8 +5887,8 @@ do_append(UnpicklerObject *self, Py_ssize_t x) static int load_append(UnpicklerObject *self) { - if (Py_SIZE(self->stack) - 1 <= 0) - return stack_underflow(); + if (Py_SIZE(self->stack) - 1 <= self->stack->fence) + return Pdata_stack_underflow(self->stack); return do_append(self, Py_SIZE(self->stack) - 1); } @@ -5886,8 +5910,8 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) int status = 0; len = Py_SIZE(self->stack); - if (x > len || x <= 0) - return stack_underflow(); + if (x > len || x <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { @@ -5940,8 +5964,8 @@ load_additems(UnpicklerObject *self) if (mark < 0) return -1; len = Py_SIZE(self->stack); - if (mark > len || mark <= 0) - return stack_underflow(); + if (mark > len || mark <= self->stack->fence) + return Pdata_stack_underflow(self->stack); if (len == mark) /* nothing to do */ return 0; @@ -5996,8 +6020,8 @@ load_build(UnpicklerObject *self) /* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.__setstate__(state). */ - if (Py_SIZE(self->stack) < 2) - return stack_underflow(); + if (Py_SIZE(self->stack) - 2 < self->stack->fence) + return Pdata_stack_underflow(self->stack); PDATA_POP(self->stack, state); if (state == NULL) @@ -6133,7 +6157,8 @@ load_mark(UnpicklerObject *self) self->marks_size = (Py_ssize_t)alloc; } - self->marks[self->num_marks++] = Py_SIZE(self->stack); + self->stack->mark_set = 1; + self->marks[self->num_marks++] = self->stack->fence = Py_SIZE(self->stack); return 0; } @@ -6216,6 +6241,8 @@ load(UnpicklerObject *self) char *s = NULL; self->num_marks = 0; + self->stack->mark_set = 0; + self->stack->fence = 0; self->proto = 0; if (Py_SIZE(self->stack)) Pdata_clear(self->stack, 0); -- cgit v0.12 From 9ec5e25f26a490510bb5da5c26a276cd30a263a0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Dec 2015 02:31:11 +0200 Subject: Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. ElementTree.XMLParser._setevents now accepts any objects with the append method, not just a list. --- Lib/xml/etree/ElementTree.py | 92 ++++++++++++++--------------------------- Misc/NEWS | 2 + Modules/_elementtree.c | 35 +++++++++------- Modules/clinic/_elementtree.c.h | 7 ++-- 4 files changed, 56 insertions(+), 80 deletions(-) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 62b5d3a..a58ef31 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -95,6 +95,7 @@ import sys import re import warnings import io +import collections import contextlib from . import ElementPath @@ -1198,16 +1199,37 @@ def iterparse(source, events=None, parser=None): Returns an iterator providing (event, elem) pairs. """ + # Use the internal, undocumented _parser argument for now; When the + # parser argument of iterparse is removed, this can be killed. + pullparser = XMLPullParser(events=events, _parser=parser) + def iterator(): + try: + while True: + yield from pullparser.read_events() + # load event buffer + data = source.read(16 * 1024) + if not data: + break + pullparser.feed(data) + root = pullparser._close_and_return_root() + yield from pullparser.read_events() + it.root = root + finally: + if close_source: + source.close() + + class IterParseIterator(collections.Iterator): + __next__ = iterator().__next__ + it = IterParseIterator() + it.root = None + del iterator, IterParseIterator + close_source = False if not hasattr(source, "read"): source = open(source, "rb") close_source = True - try: - return _IterParseIterator(source, events, parser, close_source) - except: - if close_source: - source.close() - raise + + return it class XMLPullParser: @@ -1217,9 +1239,7 @@ class XMLPullParser: # upon in user code. It will be removed in a future release. # See http://bugs.python.org/issue17741 for more details. - # _elementtree.c expects a list, not a deque - self._events_queue = [] - self._index = 0 + self._events_queue = collections.deque() self._parser = _parser or XMLParser(target=TreeBuilder()) # wire up the parser for event reporting if events is None: @@ -1257,64 +1277,14 @@ class XMLPullParser: retrieved from the iterator. """ events = self._events_queue - while True: - index = self._index - try: - event = events[self._index] - # Avoid retaining references to past events - events[self._index] = None - except IndexError: - break - index += 1 - # Compact the list in a O(1) amortized fashion - # As noted above, _elementree.c needs a list, not a deque - if index * 2 >= len(events): - events[:index] = [] - self._index = 0 - else: - self._index = index + while events: + event = events.popleft() if isinstance(event, Exception): raise event else: yield event -class _IterParseIterator: - - def __init__(self, source, events, parser, close_source=False): - # Use the internal, undocumented _parser argument for now; When the - # parser argument of iterparse is removed, this can be killed. - self._parser = XMLPullParser(events=events, _parser=parser) - self._file = source - self._close_file = close_source - self.root = self._root = None - - def __next__(self): - try: - while 1: - for event in self._parser.read_events(): - return event - if self._parser._parser is None: - break - # load event buffer - data = self._file.read(16 * 1024) - if data: - self._parser.feed(data) - else: - self._root = self._parser._close_and_return_root() - self.root = self._root - except: - if self._close_file: - self._file.close() - raise - if self._close_file: - self._file.close() - raise StopIteration - - def __iter__(self): - return self - - def XML(text, parser=None): """Parse XML document from string constant. diff --git a/Misc/NEWS b/Misc/NEWS index 578d2a3..d235f64 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,8 @@ Core and Builtins Library ------- +- Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. + - Issue #25761: Improved detecting errors in broken pickle data. - Issue #25717: Restore the previous behaviour of tolerating most fstat() diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 9026585..168c7d6 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2289,7 +2289,7 @@ typedef struct { PyObject *element_factory; /* element tracing */ - PyObject *events; /* list of events, or NULL if not collecting */ + PyObject *events_append; /* the append method of the list of events, or NULL */ PyObject *start_event_obj; /* event objects (NULL to ignore) */ PyObject *end_event_obj; PyObject *start_ns_event_obj; @@ -2324,7 +2324,7 @@ treebuilder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } t->index = 0; - t->events = NULL; + t->events_append = NULL; t->start_event_obj = t->end_event_obj = NULL; t->start_ns_event_obj = t->end_ns_event_obj = NULL; } @@ -2374,7 +2374,7 @@ treebuilder_gc_clear(TreeBuilderObject *self) Py_CLEAR(self->start_ns_event_obj); Py_CLEAR(self->end_event_obj); Py_CLEAR(self->start_event_obj); - Py_CLEAR(self->events); + Py_CLEAR(self->events_append); Py_CLEAR(self->stack); Py_CLEAR(self->data); Py_CLEAR(self->last); @@ -2455,13 +2455,14 @@ treebuilder_append_event(TreeBuilderObject *self, PyObject *action, PyObject *node) { if (action != NULL) { - PyObject *res = PyTuple_Pack(2, action, node); - if (res == NULL) + PyObject *res; + PyObject *event = PyTuple_Pack(2, action, node); + if (event == NULL) return -1; - if (PyList_Append(self->events, res) < 0) { - Py_DECREF(res); + res = PyObject_CallFunctionObjArgs(self->events_append, event, NULL); + Py_DECREF(event); + if (res == NULL) return -1; - } Py_DECREF(res); } return 0; @@ -3039,7 +3040,7 @@ expat_start_ns_handler(XMLParserObject* self, const XML_Char* prefix, if (PyErr_Occurred()) return; - if (!target->events || !target->start_ns_event_obj) + if (!target->events_append || !target->start_ns_event_obj) return; if (!uri) @@ -3062,7 +3063,7 @@ expat_end_ns_handler(XMLParserObject* self, const XML_Char* prefix_in) if (PyErr_Occurred()) return; - if (!target->events) + if (!target->events_append) return; treebuilder_append_event(target, target->end_ns_event_obj, Py_None); @@ -3551,7 +3552,7 @@ _elementtree_XMLParser_doctype_impl(XMLParserObject *self, PyObject *name, /*[clinic input] _elementtree.XMLParser._setevents - events_queue: object(subclass_of='&PyList_Type') + events_queue: object events_to_report: object = None / @@ -3561,12 +3562,12 @@ static PyObject * _elementtree_XMLParser__setevents_impl(XMLParserObject *self, PyObject *events_queue, PyObject *events_to_report) -/*[clinic end generated code: output=1440092922b13ed1 input=59db9742910c6174]*/ +/*[clinic end generated code: output=1440092922b13ed1 input=abf90830a1c3b0fc]*/ { /* activate element event reporting */ Py_ssize_t i, seqlen; TreeBuilderObject *target; - PyObject *events_seq; + PyObject *events_append, *events_seq; if (!TreeBuilder_CheckExact(self->target)) { PyErr_SetString( @@ -3579,9 +3580,11 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, target = (TreeBuilderObject*) self->target; - Py_INCREF(events_queue); - Py_XDECREF(target->events); - target->events = events_queue; + events_append = PyObject_GetAttrString(events_queue, "append"); + if (events_append == NULL) + return NULL; + Py_XDECREF(target->events_append); + target->events_append = events_append; /* clear out existing events */ Py_CLEAR(target->start_event_obj); diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 86b4c4c..92e98cf 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -668,12 +668,13 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *args) PyObject *events_queue; PyObject *events_to_report = Py_None; - if (!PyArg_ParseTuple(args, "O!|O:_setevents", - &PyList_Type, &events_queue, &events_to_report)) + if (!PyArg_UnpackTuple(args, "_setevents", + 1, 2, + &events_queue, &events_to_report)) goto exit; return_value = _elementtree_XMLParser__setevents_impl(self, events_queue, events_to_report); exit: return return_value; } -/*[clinic end generated code: output=25b8bf7e7f2151ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=19d94e2d2726d3aa input=a9049054013a1b77]*/ -- cgit v0.12 From 3ac5380d249ea572ae9f73c7ab65180c2c06a8d4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 7 Dec 2015 11:32:00 +0200 Subject: Issue #25761: Fixed reference leak added in previous changeset (5c670af0100f). --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 38598c5..b3d9b91 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5152,6 +5152,7 @@ load_dict(UnpicklerObject *self) if ((j - i) % 2 != 0) { PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); + Py_DECREF(dict); return -1; } -- cgit v0.12 From 2300bf29e69ec9ecee563134f8cdfdab93b2a7c0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 7 Dec 2015 20:45:16 -0800 Subject: Only update the arr variable when PyObject_RichCompareBool() has been called. --- Modules/_heapqmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 28604af..1c37c75 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -78,6 +78,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) if (cmp < 0) return -1; childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */ + arr = _PyList_ITEMS(heap); /* arr may have changed */ if (endpos != PyList_GET_SIZE(heap)) { PyErr_SetString(PyExc_RuntimeError, "list changed size during iteration"); @@ -85,7 +86,6 @@ siftup(PyListObject *heap, Py_ssize_t pos) } } /* Move the smaller child up. */ - arr = _PyList_ITEMS(heap); tmp1 = arr[childpos]; tmp2 = arr[pos]; arr[childpos] = tmp2; @@ -432,6 +432,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) if (cmp < 0) return -1; childpos += ((unsigned)cmp ^ 1); /* increment when cmp==0 */ + arr = _PyList_ITEMS(heap); /* arr may have changed */ if (endpos != PyList_GET_SIZE(heap)) { PyErr_SetString(PyExc_RuntimeError, "list changed size during iteration"); @@ -439,7 +440,6 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } } /* Move the smaller child up. */ - arr = _PyList_ITEMS(heap); tmp1 = arr[childpos]; tmp2 = arr[pos]; arr[childpos] = tmp2; -- cgit v0.12 From 36ff997988bdf2a72f362e60b7b03f439506411c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 10 Dec 2015 09:51:53 +0200 Subject: Issue #25638: Optimized ElementTree parsing; it is now 10% faster. --- Misc/NEWS | 1 + Modules/_elementtree.c | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 67cd7bb..372c355 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -110,6 +110,7 @@ Library ------- - Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. + Optimized ElementTree parsing; it is now 10% faster. - Issue #25761: Improved detecting errors in broken pickle data. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 3cf3d59..c483d87 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2491,10 +2491,17 @@ treebuilder_handle_start(TreeBuilderObject* self, PyObject* tag, self->data = NULL; } - if (self->element_factory && self->element_factory != Py_None) { - node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); - } else { + if (!self->element_factory || self->element_factory == Py_None) { node = create_new_element(tag, attrib); + } else if (attrib == Py_None) { + attrib = PyDict_New(); + if (!attrib) + return NULL; + node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); + Py_DECREF(attrib); + } + else { + node = PyObject_CallFunction(self->element_factory, "OO", tag, attrib); } if (!node) { return NULL; @@ -2959,12 +2966,8 @@ expat_start_handler(XMLParserObject* self, const XML_Char* tag_in, attrib_in += 2; } } else { - /* Pass an empty dictionary on */ - attrib = PyDict_New(); - if (!attrib) { - Py_DECREF(tag); - return; - } + Py_INCREF(Py_None); + attrib = Py_None; } if (TreeBuilder_CheckExact(self->target)) { @@ -2973,6 +2976,14 @@ expat_start_handler(XMLParserObject* self, const XML_Char* tag_in, tag, attrib); } else if (self->handle_start) { + if (attrib == Py_None) { + Py_DECREF(attrib); + attrib = PyDict_New(); + if (!attrib) { + Py_DECREF(tag); + return; + } + } res = PyObject_CallFunction(self->handle_start, "OO", tag, attrib); } else res = NULL; -- cgit v0.12 From 5088f6005ff5e6bb9fbc1bdfd9777b827c6aab0f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Dec 2015 18:45:01 -0800 Subject: Hoist constant expressions (so->table and so->mask) out of the inner-loop. --- Objects/setobject.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 7fde01f..b00e85f 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -260,16 +260,13 @@ The caller is responsible for updating the key's reference count and the setobject's fill and used fields. */ static void -set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) +set_insert_clean(setentry *table, size_t mask, PyObject *key, Py_hash_t hash) { - setentry *table = so->table; setentry *entry; size_t perturb = hash; - size_t mask = (size_t)so->mask; size_t i = (size_t)hash & mask; size_t j; - assert(so->fill == so->used); while (1) { entry = &table[i]; if (entry->key == NULL) @@ -285,8 +282,8 @@ set_insert_clean(PySetObject *so, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; } found_null: - entry->key = key; entry->hash = hash; + entry->key = key; } /* ======== End logic for probing the hash table ========================== */ @@ -304,6 +301,8 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) setentry *oldtable, *newtable, *entry; Py_ssize_t oldfill = so->fill; Py_ssize_t oldused = so->used; + Py_ssize_t oldmask = so->mask; + size_t newmask; int is_oldtable_malloced; setentry small_copy[PySet_MINSIZE]; @@ -363,18 +362,17 @@ set_table_resize(PySetObject *so, Py_ssize_t minused) /* Copy the data over; this is refcount-neutral for active entries; dummy entries aren't copied over, of course */ + newmask = (size_t)so->mask; if (oldfill == oldused) { - for (entry = oldtable; oldused > 0; entry++) { + for (entry = oldtable; entry <= oldtable + oldmask; entry++) { if (entry->key != NULL) { - oldused--; - set_insert_clean(so, entry->key, entry->hash); + set_insert_clean(newtable, newmask, entry->key, entry->hash); } } } else { - for (entry = oldtable; oldused > 0; entry++) { + for (entry = oldtable; entry <= oldtable + oldmask; entry++) { if (entry->key != NULL && entry->key != dummy) { - oldused--; - set_insert_clean(so, entry->key, entry->hash); + set_insert_clean(newtable, newmask, entry->key, entry->hash); } } } @@ -676,13 +674,15 @@ set_merge(PySetObject *so, PyObject *otherset) /* If our table is empty, we can use set_insert_clean() */ if (so->fill == 0) { + setentry *newtable = so->table; + size_t newmask = (size_t)so->mask; so->fill = other->used; so->used = other->used; for (i = 0; i <= other->mask; i++, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { Py_INCREF(key); - set_insert_clean(so, key, other_entry->hash); + set_insert_clean(newtable, newmask, key, other_entry->hash); } } return 0; -- cgit v0.12 From 86d322f0204797b0a2fe46e33bce240335270c6f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 13 Dec 2015 19:27:17 -0800 Subject: Undo inadvertent line swap --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index b00e85f..4723b58 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -282,8 +282,8 @@ set_insert_clean(setentry *table, size_t mask, PyObject *key, Py_hash_t hash) i = (i * 5 + 1 + perturb) & mask; } found_null: - entry->hash = hash; entry->key = key; + entry->hash = hash; } /* ======== End logic for probing the hash table ========================== */ -- cgit v0.12 From e4495877dd08b4d0568bb8a75ae5c69b3d91ff60 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Tue, 15 Dec 2015 00:42:30 -0800 Subject: Minor tweek. Counting down rather than up reduces register pressure. --- Objects/setobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 4723b58..45aa200 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -678,7 +678,7 @@ set_merge(PySetObject *so, PyObject *otherset) size_t newmask = (size_t)so->mask; so->fill = other->used; so->used = other->used; - for (i = 0; i <= other->mask; i++, other_entry++) { + for (i = other->mask + 1; i > 0 ; i--, other_entry++) { key = other_entry->key; if (key != NULL && key != dummy) { Py_INCREF(key); -- cgit v0.12 From 129a41449071928cc7afcb4789cfae22b08607b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Dec 2015 11:29:59 +0100 Subject: Issue #25868: Try to make test_eintr.test_sigwaitinfo() more reliable especially on slow buildbots Use a pipe to synchronize the parent and the child processes. --- Lib/test/eintrdata/eintr_tester.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index ee6e75b..531d576 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -377,10 +377,10 @@ class SignalEINTRTest(EINTRBaseTest): @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), 'need signal.sigwaitinfo()') def test_sigwaitinfo(self): - # Issue #25277: The sleep is a weak synchronization between the parent - # and the child process. If the sleep is too low, the test hangs on - # slow or highly loaded systems. - self.sleep_time = 2.0 + # Issue #25277, #25868: give a few miliseconds to the parent process + # between os.write() and signal.sigwaitinfo() to works around a race + # condition + self.sleep_time = 0.100 signum = signal.SIGUSR1 pid = os.getpid() @@ -388,18 +388,28 @@ class SignalEINTRTest(EINTRBaseTest): old_handler = signal.signal(signum, lambda *args: None) self.addCleanup(signal.signal, signum, old_handler) + rpipe, wpipe = os.pipe() + code = '\n'.join(( 'import os, time', 'pid = %s' % os.getpid(), 'signum = %s' % int(signum), 'sleep_time = %r' % self.sleep_time, + 'rpipe = %r' % rpipe, + 'os.read(rpipe, 1)', + 'os.close(rpipe)', 'time.sleep(sleep_time)', 'os.kill(pid, signum)', )) t0 = time.monotonic() - proc = self.subprocess(code) + proc = self.subprocess(code, pass_fds=(rpipe,)) + os.close(rpipe) with kill_on_error(proc): + # sync child-parent + os.write(wpipe, b'x') + os.close(wpipe) + # parent signal.sigwaitinfo([signum]) dt = time.monotonic() - t0 -- cgit v0.12 From 9b3a2eec1c64466cd2e2004b7732cf2b5a79a644 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:03:13 +0200 Subject: Issues #25890, #25891, #25892: Removed unused variables in Windows code. Reported by Alexander Riccio. --- Modules/posixmodule.c | 2 -- Objects/unicodeobject.c | 1 - 2 files changed, 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0e423f9..81c3c48 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3463,7 +3463,6 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) char *bufptr = namebuf; /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; - PyObject *po = NULL; wchar_t *wnamebuf = NULL; if (!path->narrow) { @@ -12380,7 +12379,6 @@ enable_symlink() HANDLE tok; TOKEN_PRIVILEGES tok_priv; LUID luid; - int meth_idx = 0; if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS, &tok)) return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 895a4e8..ad8f505 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -7332,7 +7332,6 @@ encode_code_page_strict(UINT code_page, PyObject **outbytes, BOOL usedDefaultChar = FALSE; BOOL *pusedDefaultChar = &usedDefaultChar; int outsize; - PyObject *exc = NULL; wchar_t *p; Py_ssize_t size; const DWORD flags = encode_code_page_flags(code_page, NULL); -- cgit v0.12 From 8bc2b4d5227871d25eeaac91cd64387cc27d6d74 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:06:58 +0200 Subject: Issue #25890: Removed yet one unused variable. --- Modules/posixmodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 81c3c48..7ec1f47 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3460,7 +3460,6 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) BOOL result; WIN32_FIND_DATA FileData; char namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ - char *bufptr = namebuf; /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; wchar_t *wnamebuf = NULL; -- cgit v0.12 From f8ed0044f62b4ce0c21ccb1873f6bf656cd8ac01 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 10:19:30 +0200 Subject: Issue #25889: Got rid of warning about mixing signed/unsigned char pointers. --- PC/launcher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PC/launcher.c b/PC/launcher.c index e414237..a94241b 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1106,7 +1106,7 @@ maybe_handle_shebang(wchar_t ** argv, wchar_t * cmdline) */ FILE * fp; errno_t rc = _wfopen_s(&fp, *argv, L"rb"); - unsigned char buffer[BUFSIZE]; + char buffer[BUFSIZE]; wchar_t shebang_line[BUFSIZE + 1]; size_t read; char *p; @@ -1128,7 +1128,8 @@ maybe_handle_shebang(wchar_t ** argv, wchar_t * cmdline) fclose(fp); if ((read >= 4) && (buffer[3] == '\n') && (buffer[2] == '\r')) { - ip = find_by_magic((buffer[1] << 8 | buffer[0]) & 0xFFFF); + ip = find_by_magic((((unsigned char)buffer[1]) << 8 | + (unsigned char)buffer[0]) & 0xFFFF); if (ip != NULL) { debug(L"script file is compiled against Python %ls\n", ip->version); -- cgit v0.12 From 42bb126f0aa1de4817babb8c56b9ad6df8bab458 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 18 Dec 2015 13:23:33 +0200 Subject: Issue #25899: Converted Objects/listsort.txt to UTF-8. Original patch by Chris Angelico. --- Objects/listsort.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 832e4f2..fef982f 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -486,7 +486,7 @@ sub-run, yet finding such very efficiently when they exist. I first learned about the galloping strategy in a related context; see: "Adaptive Set Intersections, Unions, and Differences" (2000) - Erik D. Demaine, Alejandro López-Ortiz, J. Ian Munro + Erik D. Demaine, Alejandro López-Ortiz, J. Ian Munro and its followup(s). An earlier paper called the same strategy "exponential search": -- cgit v0.12 From a254921cd4c1ca22d725872601292c48b7caa20a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 19 Dec 2015 09:43:14 +0200 Subject: Issue #22227: The TarFile iterator is reimplemented using generator. This implementation is simpler that using class. --- Lib/tarfile.py | 68 ++++++++++++++++++++++------------------------------------ Misc/NEWS | 3 +++ 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 9d22c8e..80b6e35 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2372,9 +2372,32 @@ class TarFile(object): """Provide an iterator object. """ if self._loaded: - return iter(self.members) - else: - return TarIter(self) + yield from self.members + return + + # Yield items using TarFile's next() method. + # When all members have been read, set TarFile as _loaded. + index = 0 + # Fix for SF #1100429: Under rare circumstances it can + # happen that getmembers() is called during iteration, + # which will have already exhausted the next() method. + if self.firstmember is not None: + tarinfo = self.next() + index += 1 + yield tarinfo + + while True: + if index < len(self.members): + tarinfo = self.members[index] + elif not self._loaded: + tarinfo = self.next() + if not tarinfo: + self._loaded = True + return + else: + return + index += 1 + yield tarinfo def _dbg(self, level, msg): """Write debugging output to sys.stderr. @@ -2395,45 +2418,6 @@ class TarFile(object): if not self._extfileobj: self.fileobj.close() self.closed = True -# class TarFile - -class TarIter: - """Iterator Class. - - for tarinfo in TarFile(...): - suite... - """ - - def __init__(self, tarfile): - """Construct a TarIter object. - """ - self.tarfile = tarfile - self.index = 0 - def __iter__(self): - """Return iterator object. - """ - return self - def __next__(self): - """Return the next item using TarFile's next() method. - When all members have been read, set TarFile as _loaded. - """ - # Fix for SF #1100429: Under rare circumstances it can - # happen that getmembers() is called during iteration, - # which will cause TarIter to stop prematurely. - - if self.index == 0 and self.tarfile.firstmember is not None: - tarinfo = self.tarfile.next() - elif self.index < len(self.tarfile.members): - tarinfo = self.tarfile.members[self.index] - elif not self.tarfile._loaded: - tarinfo = self.tarfile.next() - if not tarinfo: - self.tarfile._loaded = True - raise StopIteration - else: - raise StopIteration - self.index += 1 - return tarinfo #-------------------- # exported functions diff --git a/Misc/NEWS b/Misc/NEWS index b57dbf1..92572fa 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -109,6 +109,9 @@ Core and Builtins Library ------- +- Issue #22227: The TarFile iterator is reimplemented using generator. + This implementation is simpler that using class. + - Issue #25638: Optimized ElementTree.iterparse(); it is now 2x faster. Optimized ElementTree parsing; it is now 10% faster. -- cgit v0.12 From 22adf2ac022d6667a8689d6fac8af44c7663e3f2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Dec 2015 12:43:54 +0200 Subject: Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. --- Misc/NEWS | 4 + Modules/_elementtree.c | 259 +++++++++++++++++++------------------------------ 2 files changed, 105 insertions(+), 158 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index ad37eb7..6dc02e3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,10 @@ Core and Builtins Library ------- +- Issue #25873: Optimized iterating ElementTree. Iterating elements + Element.iter() is now 40% faster, iterating text Element.itertext() + is now up to 2.5 times faster. + - Issue #25902: Fixed various refcount issues in ElementTree iteration. - Issue #22227: The TarFile iterator is reimplemented using generator. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 949e3a0..ad3e45c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1986,22 +1986,22 @@ static PySequenceMethods element_as_sequence = { * pre-order traversal. To keep track of which sub-element should be returned * next, a stack of parents is maintained. This is a standard stack-based * iterative pre-order traversal of a tree. - * The stack is managed using a single-linked list starting at parent_stack. - * Each stack node contains the saved parent to which we should return after + * The stack is managed using a continuous array. + * Each stack item contains the saved parent to which we should return after * the current one is exhausted, and the next child to examine in that parent. */ typedef struct ParentLocator_t { ElementObject *parent; Py_ssize_t child_index; - struct ParentLocator_t *next; } ParentLocator; typedef struct { PyObject_HEAD ParentLocator *parent_stack; + Py_ssize_t parent_stack_used; + Py_ssize_t parent_stack_size; ElementObject *root_element; PyObject *sought_tag; - int root_done; int gettext; } ElementIterObject; @@ -2009,13 +2009,11 @@ typedef struct { static void elementiter_dealloc(ElementIterObject *it) { - ParentLocator *p = it->parent_stack; - while (p) { - ParentLocator *temp = p; - Py_XDECREF(p->parent); - p = p->next; - PyObject_Free(temp); - } + Py_ssize_t i = it->parent_stack_used; + it->parent_stack_used = 0; + while (i--) + Py_XDECREF(it->parent_stack[i].parent); + PyMem_Free(it->parent_stack); Py_XDECREF(it->sought_tag); Py_XDECREF(it->root_element); @@ -2027,11 +2025,9 @@ elementiter_dealloc(ElementIterObject *it) static int elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) { - ParentLocator *p = it->parent_stack; - while (p) { - Py_VISIT(p->parent); - p = p->next; - } + Py_ssize_t i = it->parent_stack_used; + while (i--) + Py_VISIT(it->parent_stack[i].parent); Py_VISIT(it->root_element); Py_VISIT(it->sought_tag); @@ -2040,17 +2036,25 @@ elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) /* Helper function for elementiter_next. Add a new parent to the parent stack. */ -static ParentLocator * -parent_stack_push_new(ParentLocator *stack, ElementObject *parent) +static int +parent_stack_push_new(ElementIterObject *it, ElementObject *parent) { - ParentLocator *new_node = PyObject_Malloc(sizeof(ParentLocator)); - if (new_node) { - new_node->parent = parent; - Py_INCREF(parent); - new_node->child_index = 0; - new_node->next = stack; + ParentLocator *item; + + if (it->parent_stack_used >= it->parent_stack_size) { + Py_ssize_t new_size = it->parent_stack_size * 2; /* never overflow */ + ParentLocator *parent_stack = it->parent_stack; + PyMem_Resize(parent_stack, ParentLocator, new_size); + if (parent_stack == NULL) + return -1; + it->parent_stack = parent_stack; + it->parent_stack_size = new_size; } - return new_node; + item = it->parent_stack + it->parent_stack_used++; + Py_INCREF(parent); + item->parent = parent; + item->child_index = 0; + return 0; } static PyObject * @@ -2067,151 +2071,91 @@ elementiter_next(ElementIterObject *it) * - itertext() also has to handle tail, after finishing with all the * children of a node. */ - ElementObject *cur_parent; - Py_ssize_t child_index; int rc; ElementObject *elem; + PyObject *text; while (1) { /* Handle the case reached in the beginning and end of iteration, where - * the parent stack is empty. The root_done flag gives us indication - * whether we've just started iterating (so root_done is 0), in which - * case the root is returned. If root_done is 1 and we're here, the + * the parent stack is empty. If root_element is NULL and we're here, the * iterator is exhausted. */ - if (!it->parent_stack->parent) { - if (it->root_done) { + if (!it->parent_stack_used) { + if (!it->root_element) { PyErr_SetNone(PyExc_StopIteration); return NULL; - } else { - elem = it->root_element; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } + } - Py_INCREF(elem); - it->root_done = 1; - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - return (PyObject *)elem; - } - } - else { - Py_DECREF(elem); + elem = it->root_element; /* steals a reference */ + it->root_element = NULL; + } + else { + /* See if there are children left to traverse in the current parent. If + * yes, visit the next child. If not, pop the stack and try again. + */ + ParentLocator *item = &it->parent_stack[it->parent_stack_used - 1]; + Py_ssize_t child_index = item->child_index; + ElementObjectExtra *extra; + elem = item->parent; + extra = elem->extra; + if (!extra || child_index >= extra->length) { + it->parent_stack_used--; + /* Note that extra condition on it->parent_stack_used here; + * this is because itertext() is supposed to only return *inner* + * text, not text following the element it began iteration with. + */ + if (it->gettext && it->parent_stack_used) { + text = element_get_tail(elem); + goto gettext; } + Py_DECREF(elem); + continue; } + + elem = (ElementObject *)extra->children[child_index]; + item->child_index++; + Py_INCREF(elem); } - /* See if there are children left to traverse in the current parent. If - * yes, visit the next child. If not, pop the stack and try again. - */ - cur_parent = it->parent_stack->parent; - child_index = it->parent_stack->child_index; - if (cur_parent->extra && child_index < cur_parent->extra->length) { - elem = (ElementObject *)cur_parent->extra->children[child_index]; - it->parent_stack->child_index++; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } + if (parent_stack_push_new(it, elem) < 0) { + Py_DECREF(elem); + PyErr_NoMemory(); + return NULL; + } + if (it->gettext) { + text = element_get_text(elem); + goto gettext; + } - Py_INCREF(elem); - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - return (PyObject *)elem; - } - Py_DECREF(elem); - } + if (it->sought_tag == Py_None) + return (PyObject *)elem; + + rc = PyObject_RichCompareBool(elem->tag, it->sought_tag, Py_EQ); + if (rc > 0) + return (PyObject *)elem; + + Py_DECREF(elem); + if (rc < 0) + return NULL; + continue; + +gettext: + if (!text) { + Py_DECREF(elem); + return NULL; + } + if (text == Py_None) { + Py_DECREF(elem); } else { - PyObject *tail; - ParentLocator *next; - if (it->gettext) { - Py_INCREF(cur_parent); - tail = element_get_tail(cur_parent); - if (!tail) { - Py_DECREF(cur_parent); - return NULL; - } - Py_INCREF(tail); - Py_DECREF(cur_parent); - } - else { - tail = Py_None; - Py_INCREF(tail); - } - next = it->parent_stack->next; - cur_parent = it->parent_stack->parent; - PyObject_Free(it->parent_stack); - it->parent_stack = next; - Py_XDECREF(cur_parent); - - /* Note that extra condition on it->parent_stack->parent here; - * this is because itertext() is supposed to only return *inner* - * text, not text following the element it began iteration with. - */ - if (it->parent_stack->parent) { - rc = PyObject_IsTrue(tail); - if (rc > 0) - return tail; - Py_DECREF(tail); - if (rc < 0) - return NULL; - } - else { - Py_DECREF(tail); - } + Py_INCREF(text); + Py_DECREF(elem); + rc = PyObject_IsTrue(text); + if (rc > 0) + return text; + Py_DECREF(text); + if (rc < 0) + return NULL; } } @@ -2263,6 +2207,7 @@ static PyTypeObject ElementIter_Type = { 0, /* tp_new */ }; +#define INIT_PARENT_STACK_SIZE 8 static PyObject * create_elementiter(ElementObject *self, PyObject *tag, int gettext) @@ -2275,22 +2220,20 @@ create_elementiter(ElementObject *self, PyObject *tag, int gettext) Py_INCREF(tag); it->sought_tag = tag; - it->root_done = 0; it->gettext = gettext; Py_INCREF(self); it->root_element = self; PyObject_GC_Track(it); - it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)); + it->parent_stack = PyMem_New(ParentLocator, INIT_PARENT_STACK_SIZE); if (it->parent_stack == NULL) { Py_DECREF(it); PyErr_NoMemory(); return NULL; } - it->parent_stack->parent = NULL; - it->parent_stack->child_index = 0; - it->parent_stack->next = NULL; + it->parent_stack_used = 0; + it->parent_stack_size = INIT_PARENT_STACK_SIZE; return (PyObject *)it; } -- cgit v0.12 From 060ed718cecbfb7bb3292b76eef8312896ba7317 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Dec 2015 12:57:27 +0200 Subject: Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. --- Misc/NEWS | 2 ++ Modules/_elementtree.c | 78 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 6dc02e3..ff1beef 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,8 @@ Core and Builtins Library ------- +- Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. + - Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index ad3e45c..5908c72 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -129,30 +129,6 @@ elementtree_free(void *m) /* helpers */ LOCAL(PyObject*) -deepcopy(PyObject* object, PyObject* memo) -{ - /* do a deep copy of the given object */ - PyObject* args; - PyObject* result; - elementtreestate *st = ET_STATE_GLOBAL; - - if (!st->deepcopy_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "deepcopy helper not found" - ); - return NULL; - } - - args = PyTuple_Pack(2, object, memo); - if (!args) - return NULL; - result = PyObject_CallObject(st->deepcopy_obj, args); - Py_DECREF(args); - return result; -} - -LOCAL(PyObject*) list_join(PyObject* list) { /* join list elements (destroying the list in the process) */ @@ -748,6 +724,9 @@ _elementtree_Element___copy___impl(ElementObject *self) return (PyObject*) element; } +/* Helper for a deep copy. */ +LOCAL(PyObject *) deepcopy(PyObject *, PyObject *); + /*[clinic input] _elementtree.Element.__deepcopy__ @@ -838,6 +817,57 @@ _elementtree_Element___deepcopy__(ElementObject *self, PyObject *memo) return NULL; } +LOCAL(PyObject *) +deepcopy(PyObject *object, PyObject *memo) +{ + /* do a deep copy of the given object */ + PyObject *args; + PyObject *result; + elementtreestate *st; + + /* Fast paths */ + if (object == Py_None || PyUnicode_CheckExact(object)) { + Py_INCREF(object); + return object; + } + + if (Py_REFCNT(object) == 1) { + if (PyDict_CheckExact(object)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + int simple = 1; + while (PyDict_Next(object, &pos, &key, &value)) { + if (!PyUnicode_CheckExact(key) || !PyUnicode_CheckExact(value)) { + simple = 0; + break; + } + } + if (simple) + return PyDict_Copy(object); + /* Fall through to general case */ + } + else if (Element_CheckExact(object)) { + return _elementtree_Element___deepcopy__((ElementObject *)object, memo); + } + } + + /* General case */ + st = ET_STATE_GLOBAL; + if (!st->deepcopy_obj) { + PyErr_SetString(PyExc_RuntimeError, + "deepcopy helper not found"); + return NULL; + } + + args = PyTuple_Pack(2, object, memo); + if (!args) + return NULL; + result = PyObject_CallObject(st->deepcopy_obj, args); + Py_DECREF(args); + return result; +} + + /*[clinic input] _elementtree.Element.__sizeof__ -> Py_ssize_t -- cgit v0.12 From 2d06e8445587d9b4d0bf79bdb08ab4743b780249 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 25 Dec 2015 19:53:18 +0200 Subject: Issue #25923: Added the const qualifier to static constant arrays. --- Include/py_curses.h | 4 ++-- Include/pymath.h | 2 +- Modules/_csv.c | 8 ++++---- Modules/_ctypes/_ctypes.c | 14 +++++++------- Modules/_ctypes/callproc.c | 16 ++++++++-------- Modules/_curses_panel.c | 2 +- Modules/_datetimemodule.c | 20 ++++++++++---------- Modules/_dbmmodule.c | 8 ++++---- Modules/_gdbmmodule.c | 2 +- Modules/_io/textio.c | 4 ++-- Modules/_randommodule.c | 2 +- Modules/_sqlite/connection.c | 2 +- Modules/_sqlite/cursor.c | 4 ++-- Modules/_sqlite/module.c | 4 ++-- Modules/_sre.c | 2 +- Modules/_struct.c | 4 ++-- Modules/_testbuffer.c | 2 +- Modules/_testcapimodule.c | 4 ++-- Modules/arraymodule.c | 18 +++++++++--------- Modules/audioop.c | 20 +++++++++++--------- Modules/binascii.c | 14 +++++++------- Modules/getaddrinfo.c | 2 +- Modules/getpath.c | 4 ++-- Modules/main.c | 14 +++++++------- Modules/parsermodule.c | 4 ++-- Modules/posixmodule.c | 10 +++++----- Modules/selectmodule.c | 2 +- Modules/timemodule.c | 4 ++-- Modules/unicodedata.c | 4 ++-- Objects/object.c | 2 +- Objects/structseq.c | 6 +++--- Objects/typeobject.c | 2 +- Objects/unicodeobject.c | 12 ++++++------ Parser/parsetok.c | 4 ++-- Parser/pgen.c | 2 +- Python/ast.c | 4 ++-- Python/dtoa.c | 4 ++-- Python/formatter_unicode.c | 2 +- Python/import.c | 6 +++--- Python/importdl.c | 4 ++-- Python/mystrtoul.c | 6 +++--- Python/pystrtod.c | 9 +++++---- Python/pythonrun.c | 4 ++-- Python/sysmodule.c | 6 ++++-- 44 files changed, 139 insertions(+), 134 deletions(-) diff --git a/Include/py_curses.h b/Include/py_curses.h index f2c08f6..3c21697 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -103,8 +103,8 @@ static void **PyCurses_API; #endif /* general error messages */ -static char *catchall_ERR = "curses function returned ERR"; -static char *catchall_NULL = "curses function returned NULL"; +static const char catchall_ERR[] = "curses function returned ERR"; +static const char catchall_NULL[] = "curses function returned NULL"; /* Function Prototype Macros - They are ugly but very, very useful. ;-) diff --git a/Include/pymath.h b/Include/pymath.h index 1ea9ac1..ed76053 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -169,7 +169,7 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); #pragma float_control (pop) #define Py_NAN __icc_nan() #else /* ICC_NAN_RELAXED as default for Intel Compiler */ - static union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; + static const union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; #define Py_NAN (__nan_store.__icc_nan) #endif /* ICC_NAN_STRICT */ #endif /* __INTEL_COMPILER */ diff --git a/Modules/_csv.c b/Modules/_csv.c index fe85069..c0be739 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -60,10 +60,10 @@ typedef enum { typedef struct { QuoteStyle style; - char *name; + const char *name; } StyleDesc; -static StyleDesc quote_styles[] = { +static const StyleDesc quote_styles[] = { { QUOTE_MINIMAL, "QUOTE_MINIMAL" }, { QUOTE_ALL, "QUOTE_ALL" }, { QUOTE_NONNUMERIC, "QUOTE_NONNUMERIC" }, @@ -286,7 +286,7 @@ _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) static int dialect_check_quoting(int quoting) { - StyleDesc *qs; + const StyleDesc *qs; for (qs = quote_styles; qs->name; qs++) { if ((int)qs->style == quoting) @@ -1633,7 +1633,7 @@ PyMODINIT_FUNC PyInit__csv(void) { PyObject *module; - StyleDesc *style; + const StyleDesc *style; if (PyType_Ready(&Dialect_Type) < 0) return NULL; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 34a1099..fccc8ba 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -435,7 +435,7 @@ UnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return StructUnionType_new(type, args, kwds, 0); } -static char from_address_doc[] = +static const char from_address_doc[] = "C.from_address(integer) -> C instance\naccess a C instance at the specified address"; static PyObject * @@ -453,7 +453,7 @@ CDataType_from_address(PyObject *type, PyObject *value) return PyCData_AtAddress(type, buf); } -static char from_buffer_doc[] = +static const char from_buffer_doc[] = "C.from_buffer(object, offset=0) -> C instance\ncreate a C instance from a writeable buffer"; static int @@ -524,7 +524,7 @@ CDataType_from_buffer(PyObject *type, PyObject *args) return result; } -static char from_buffer_copy_doc[] = +static const char from_buffer_copy_doc[] = "C.from_buffer_copy(object, offset=0) -> C instance\ncreate a C instance from a readable buffer"; static PyObject * @@ -566,7 +566,7 @@ CDataType_from_buffer_copy(PyObject *type, PyObject *args) return result; } -static char in_dll_doc[] = +static const char in_dll_doc[] = "C.in_dll(dll, name) -> C instance\naccess a C instance in a dll"; static PyObject * @@ -623,7 +623,7 @@ CDataType_in_dll(PyObject *type, PyObject *args) return PyCData_AtAddress(type, address); } -static char from_param_doc[] = +static const char from_param_doc[] = "Convert a Python object into a function call parameter."; static PyObject * @@ -1481,7 +1481,7 @@ _type_ attribute. */ -static char *SIMPLE_TYPE_CHARS = "cbBhHiIlLdfuzZqQPXOv?g"; +static const char SIMPLE_TYPE_CHARS[] = "cbBhHiIlLdfuzZqQPXOv?g"; static PyObject * c_wchar_p_from_param(PyObject *type, PyObject *value) @@ -5118,7 +5118,7 @@ static const char module_docs[] = #ifdef MS_WIN32 -static char comerror_doc[] = "Raised when a COM method call failed."; +static const char comerror_doc[] = "Raised when a COM method call failed."; int comerror_init(PyObject *self, PyObject *args, PyObject *kwds) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 03a911f..68981fe 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1201,7 +1201,7 @@ _parse_voidp(PyObject *obj, void **address) #ifdef MS_WIN32 -static char format_error_doc[] = +static const char format_error_doc[] = "FormatError([integer]) -> string\n\ \n\ Convert a win32 error code into a string. If the error code is not\n\ @@ -1225,7 +1225,7 @@ static PyObject *format_error(PyObject *self, PyObject *args) return result; } -static char load_library_doc[] = +static const char load_library_doc[] = "LoadLibrary(name) -> handle\n\ \n\ Load an executable (usually a DLL), and return a handle to it.\n\ @@ -1254,7 +1254,7 @@ static PyObject *load_library(PyObject *self, PyObject *args) #endif } -static char free_library_doc[] = +static const char free_library_doc[] = "FreeLibrary(handle) -> void\n\ \n\ Free the handle of an executable previously loaded by LoadLibrary.\n"; @@ -1269,7 +1269,7 @@ static PyObject *free_library(PyObject *self, PyObject *args) return Py_None; } -static char copy_com_pointer_doc[] = +static const char copy_com_pointer_doc[] = "CopyComPointer(src, dst) -> HRESULT value\n"; static PyObject * @@ -1439,7 +1439,7 @@ call_cdeclfunction(PyObject *self, PyObject *args) /***************************************************************** * functions */ -static char sizeof_doc[] = +static const char sizeof_doc[] = "sizeof(C type) -> integer\n" "sizeof(C instance) -> integer\n" "Return the size in bytes of a C instance"; @@ -1460,7 +1460,7 @@ sizeof_func(PyObject *self, PyObject *obj) return NULL; } -static char alignment_doc[] = +static const char alignment_doc[] = "alignment(C type) -> integer\n" "alignment(C instance) -> integer\n" "Return the alignment requirements of a C instance"; @@ -1483,7 +1483,7 @@ align_func(PyObject *self, PyObject *obj) return NULL; } -static char byref_doc[] = +static const char byref_doc[] = "byref(C instance[, offset=0]) -> byref-object\n" "Return a pointer lookalike to a C instance, only usable\n" "as function argument"; @@ -1527,7 +1527,7 @@ byref(PyObject *self, PyObject *args) return (PyObject *)parg; } -static char addressof_doc[] = +static const char addressof_doc[] = "addressof(C instance) -> integer\n" "Return the address of the C instance internal buffer"; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 759b731..a9c406f 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -6,7 +6,7 @@ /* Release Number */ -static char *PyCursesVersion = "2.1"; +static const char PyCursesVersion[] = "2.1"; /* Includes */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 55988c5..e8b7ae8 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -184,12 +184,12 @@ divide_nearest(PyObject *m, PyObject *n) * and the number of days before that month in the same year. These * are correct for non-leap years only. */ -static int _days_in_month[] = { +static const int _days_in_month[] = { 0, /* unused; this vector uses 1-based indexing */ 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; -static int _days_before_month[] = { +static const int _days_before_month[] = { 0, /* unused; this vector uses 1-based indexing */ 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }; @@ -1009,10 +1009,10 @@ append_keyword_tzinfo(PyObject *repr, PyObject *tzinfo) static PyObject * format_ctime(PyDateTime_Date *date, int hours, int minutes, int seconds) { - static const char *DayNames[] = { + static const char * const DayNames[] = { "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" }; - static const char *MonthNames[] = { + static const char * const MonthNames[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; @@ -2307,7 +2307,7 @@ static PyMethodDef delta_methods[] = { {NULL, NULL}, }; -static char delta_doc[] = +static const char delta_doc[] = PyDoc_STR("Difference between two datetime values."); static PyNumberMethods delta_as_number = { @@ -2886,7 +2886,7 @@ static PyMethodDef date_methods[] = { {NULL, NULL} }; -static char date_doc[] = +static const char date_doc[] = PyDoc_STR("date(year, month, day) --> date object"); static PyNumberMethods date_as_number = { @@ -3155,7 +3155,7 @@ static PyMethodDef tzinfo_methods[] = { {NULL, NULL} }; -static char tzinfo_doc[] = +static const char tzinfo_doc[] = PyDoc_STR("Abstract base class for time zone info objects."); static PyTypeObject PyDateTime_TZInfoType = { @@ -3387,7 +3387,7 @@ static PyMethodDef timezone_methods[] = { {NULL, NULL} }; -static char timezone_doc[] = +static const char timezone_doc[] = PyDoc_STR("Fixed offset from UTC implementation of tzinfo."); static PyTypeObject PyDateTime_TimeZoneType = { @@ -3877,7 +3877,7 @@ static PyMethodDef time_methods[] = { {NULL, NULL} }; -static char time_doc[] = +static const char time_doc[] = PyDoc_STR("time([hour[, minute[, second[, microsecond[, tzinfo]]]]]) --> a time object\n\ \n\ All arguments are optional. tzinfo may be None, or an instance of\n\ @@ -5065,7 +5065,7 @@ static PyMethodDef datetime_methods[] = { {NULL, NULL} }; -static char datetime_doc[] = +static const char datetime_doc[] = PyDoc_STR("datetime(year, month, day[, hour[, minute[, second[, microsecond[,tzinfo]]]]])\n\ \n\ The year, month and day arguments are required. tzinfo may be None, or an\n\ diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 02899e4..5e7ec1a 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -14,16 +14,16 @@ */ #if defined(HAVE_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; /* EMX port of GDBM */ +static const char which_dbm[] = "GNU gdbm"; /* EMX port of GDBM */ #elif defined(HAVE_GDBM_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; +static const char which_dbm[] = "GNU gdbm"; #elif defined(HAVE_GDBM_DASH_NDBM_H) #include -static char *which_dbm = "GNU gdbm"; +static const char which_dbm[] = "GNU gdbm"; #elif defined(HAVE_BERKDB_H) #include -static char *which_dbm = "Berkeley DB"; +static const char which_dbm[] = "Berkeley DB"; #else #error "No ndbm.h available!" #endif diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index f070a14..bf7b036 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -615,7 +615,7 @@ dbmopen_impl(PyModuleDef *module, const char *name, const char *flags, return newdbmobject(name, iflags, mode); } -static char dbmmodule_open_flags[] = "rwcn" +static const char dbmmodule_open_flags[] = "rwcn" #ifdef GDBM_FAST "f" #endif diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index b232b02..d018623 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -772,7 +772,7 @@ typedef struct { encodefunc_t encodefunc; } encodefuncentry; -static encodefuncentry encodefuncs[] = { +static const encodefuncentry encodefuncs[] = { {"ascii", (encodefunc_t) ascii_encode}, {"iso8859-1", (encodefunc_t) latin1_encode}, {"utf-8", (encodefunc_t) utf8_encode}, @@ -1022,7 +1022,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, goto error; } else if (PyUnicode_Check(res)) { - encodefuncentry *e = encodefuncs; + const encodefuncentry *e = encodefuncs; while (e->name != NULL) { if (!PyUnicode_CompareWithASCIIString(res, e->name)) { self->encodefunc = e->encodefunc; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 95ad4a4..a85d905 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -99,7 +99,7 @@ static PY_UINT32_T genrand_int32(RandomObject *self) { PY_UINT32_T y; - static PY_UINT32_T mag01[2]={0x0U, MATRIX_A}; + static const PY_UINT32_T mag01[2] = {0x0U, MATRIX_A}; /* mag01[x] = x * MATRIX_A for x=0,1 */ PY_UINT32_T *mt; diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7018f9f..75ed1f8 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1622,7 +1622,7 @@ pysqlite_connection_exit(pysqlite_Connection* self, PyObject* args) Py_RETURN_FALSE; } -static char connection_doc[] = +static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); static PyGetSetDef connection_getset[] = { diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index d909738..1c240d6 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -27,7 +27,7 @@ PyObject* pysqlite_cursor_iternext(pysqlite_Cursor* self); -static char* errmsg_fetch_across_rollback = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from."; +static const char errmsg_fetch_across_rollback[] = "Cursor needed to be reset because of commit/rollback and can no longer be fetched from."; static pysqlite_StatementKind detect_statement_type(const char* statement) { @@ -1050,7 +1050,7 @@ static struct PyMemberDef cursor_members[] = {NULL} }; -static char cursor_doc[] = +static const char cursor_doc[] = PyDoc_STR("SQLite database cursor class."); PyTypeObject pysqlite_CursorType = { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 7a7e860..ff2e3a5 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -261,13 +261,13 @@ static PyMethodDef module_methods[] = { }; struct _IntConstantPair { - char* constant_name; + const char *constant_name; int constant_value; }; typedef struct _IntConstantPair IntConstantPair; -static IntConstantPair _int_constants[] = { +static const IntConstantPair _int_constants[] = { {"PARSE_DECLTYPES", PARSE_DECLTYPES}, {"PARSE_COLNAMES", PARSE_COLNAMES}, diff --git a/Modules/_sre.c b/Modules/_sre.c index 919a069..f597a70 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -35,7 +35,7 @@ * other compatibility work. */ -static char copyright[] = +static const char copyright[] = " SRE 2.2.2 Copyright (c) 1997-2002 by Secret Labs AB "; #define PY_SSIZE_T_CLEAN diff --git a/Modules/_struct.c b/Modules/_struct.c index b61f9f6..b18c71d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -723,7 +723,7 @@ np_void_p(char *p, PyObject *v, const formatdef *f) return 0; } -static formatdef native_table[] = { +static const formatdef native_table[] = { {'x', sizeof(char), 0, NULL}, {'b', sizeof(char), 0, nu_byte, np_byte}, {'B', sizeof(char), 0, nu_ubyte, np_ubyte}, @@ -2280,7 +2280,7 @@ PyInit__struct(void) /* Check endian and swap in faster functions */ { - formatdef *native = native_table; + const formatdef *native = native_table; formatdef *other, *ptr; #if PY_LITTLE_ENDIAN other = lilendian_table; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 43db8a8..13d3ccc 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -13,7 +13,7 @@ PyObject *Struct = NULL; PyObject *calcsize = NULL; /* cache simple format string */ -static const char *simple_fmt = "B"; +static const char simple_fmt[] = "B"; PyObject *simple_format = NULL; #define SIMPLE_FORMAT(fmt) (fmt == NULL || strcmp(fmt, "B") == 0) #define FIX_FORMAT(fmt) (fmt == NULL ? "B" : fmt) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a896af0..cc3f52d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -887,7 +887,7 @@ static PyObject * getargs_keywords(PyObject *self, PyObject *args, PyObject *kwargs) { static char *keywords[] = {"arg1","arg2","arg3","arg4","arg5", NULL}; - static char *fmt="(ii)i|(i(ii))(iii)i"; + static const char fmt[] = "(ii)i|(i(ii))(iii)i"; int int_args[10]={-1, -1, -1, -1, -1, -1, -1, -1, -1, -1}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, fmt, keywords, @@ -3769,7 +3769,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_LONGLONG", "T_ULONGLONG", #endif NULL}; - static char *fmt = "|bbBhHiIlknfds#" + static const char fmt[] = "|bbBhHiIlknfds#" #ifdef HAVE_LONG_LONG "LK" #endif diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 6af75a4..1b0a282 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -31,7 +31,7 @@ struct arraydescr { int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); - char *formats; + const char *formats; int is_integer_type; int is_signed; }; @@ -40,7 +40,7 @@ typedef struct arrayobject { PyObject_VAR_HEAD char *ob_item; Py_ssize_t allocated; - struct arraydescr *ob_descr; + const struct arraydescr *ob_descr; PyObject *weakreflist; /* List of weak references */ int ob_exports; /* Number of exported buffers */ } arrayobject; @@ -511,7 +511,7 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) * Don't forget to update typecode_to_mformat_code() if you add a new * typecode. */ -static struct arraydescr descriptors[] = { +static const struct arraydescr descriptors[] = { {'b', 1, b_getitem, b_setitem, "b", 1, 1}, {'B', 1, BB_getitem, BB_setitem, "B", 1, 0}, {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u", 0, 0}, @@ -539,7 +539,7 @@ class array.array "arrayobject *" "&Arraytype" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=ad43d37e942a8854]*/ static PyObject * -newarrayobject(PyTypeObject *type, Py_ssize_t size, struct arraydescr *descr) +newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) { arrayobject *op; size_t nbytes; @@ -1946,7 +1946,7 @@ array__array_reconstructor_impl(PyModuleDef *module, PyTypeObject *arraytype, { PyObject *converted_items; PyObject *result; - struct arraydescr *descr; + const struct arraydescr *descr; if (!PyType_Check(arraytype)) { PyErr_Format(PyExc_TypeError, @@ -2084,7 +2084,7 @@ array__array_reconstructor_impl(PyModuleDef *module, PyTypeObject *arraytype, Py_ssize_t itemcount = Py_SIZE(items) / mf_descr.size; const unsigned char *memstr = (unsigned char *)PyBytes_AS_STRING(items); - struct arraydescr *descr; + const struct arraydescr *descr; /* If possible, try to pack array's items using a data type * that fits better. This may result in an array with narrower @@ -2554,7 +2554,7 @@ array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) view->format = NULL; view->internal = NULL; if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { - view->format = self->ob_descr->formats; + view->format = (char *)self->ob_descr->formats; #ifdef Py_UNICODE_WIDE if (self->ob_descr->typecode == 'u') { view->format = "w"; @@ -2595,7 +2595,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { int c; PyObject *initial = NULL, *it = NULL; - struct arraydescr *descr; + const struct arraydescr *descr; if (type == &Arraytype && !_PyArg_NoKeywords("array.array()", kwds)) return NULL; @@ -2987,7 +2987,7 @@ array_modexec(PyObject *m) char buffer[Py_ARRAY_LENGTH(descriptors)], *p; PyObject *typecodes; Py_ssize_t size = 0; - struct arraydescr *descr; + const struct arraydescr *descr; if (PyType_Ready(&Arraytype) < 0) return -1; diff --git a/Modules/audioop.c b/Modules/audioop.c index 3b05aec..d97a369 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -51,13 +51,15 @@ fbound(double val, double minval, double maxval) #define SEG_SHIFT (4) /* Left shift for segment number. */ #define SEG_MASK (0x70) /* Segment field mask. */ -static PyInt16 seg_aend[8] = {0x1F, 0x3F, 0x7F, 0xFF, - 0x1FF, 0x3FF, 0x7FF, 0xFFF}; -static PyInt16 seg_uend[8] = {0x3F, 0x7F, 0xFF, 0x1FF, - 0x3FF, 0x7FF, 0xFFF, 0x1FFF}; +static const PyInt16 seg_aend[8] = { + 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF +}; +static const PyInt16 seg_uend[8] = { + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF +}; static PyInt16 -search(PyInt16 val, PyInt16 *table, int size) +search(PyInt16 val, const PyInt16 *table, int size) { int i; @@ -70,7 +72,7 @@ search(PyInt16 val, PyInt16 *table, int size) #define st_ulaw2linear16(uc) (_st_ulaw2linear16[uc]) #define st_alaw2linear16(uc) (_st_alaw2linear16[uc]) -static PyInt16 _st_ulaw2linear16[256] = { +static const PyInt16 _st_ulaw2linear16[256] = { -32124, -31100, -30076, -29052, -28028, -27004, -25980, -24956, -23932, -22908, -21884, -20860, -19836, -18812, -17788, -16764, -15996, -15484, -14972, -14460, -13948, @@ -176,7 +178,7 @@ st_14linear2ulaw(PyInt16 pcm_val) /* 2's complement (14-bit range) */ } -static PyInt16 _st_alaw2linear16[256] = { +static const PyInt16 _st_alaw2linear16[256] = { -5504, -5248, -6016, -5760, -4480, -4224, -4992, -4736, -7552, -7296, -8064, -7808, -6528, -6272, -7040, -6784, -2752, -2624, -3008, -2880, -2240, @@ -270,12 +272,12 @@ st_linear2alaw(PyInt16 pcm_val) /* 2's complement (13-bit range) */ /* End of code taken from sox */ /* Intel ADPCM step variation table */ -static int indexTable[16] = { +static const int indexTable[16] = { -1, -1, -1, -1, 2, 4, 6, 8, -1, -1, -1, -1, 2, 4, 6, 8, }; -static int stepsizeTable[89] = { +static const int stepsizeTable[89] = { 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 19, 21, 23, 25, 28, 31, 34, 37, 41, 45, 50, 55, 60, 66, 73, 80, 88, 97, 107, 118, diff --git a/Modules/binascii.c b/Modules/binascii.c index ccd81fa..9df48da 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -74,7 +74,7 @@ static PyObject *Incomplete; #define SKIP 0x7E #define FAIL 0x7D -static unsigned char table_a2b_hqx[256] = { +static const unsigned char table_a2b_hqx[256] = { /* ^@ ^A ^B ^C ^D ^E ^F ^G */ /* 0*/ FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, /* \b \t \n ^K ^L \r ^N ^O */ @@ -125,10 +125,10 @@ static unsigned char table_a2b_hqx[256] = { FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, FAIL, }; -static unsigned char table_b2a_hqx[] = +static const unsigned char table_b2a_hqx[] = "!\"#$%&'()*+,-012345689@ABCDEFGHIJKLMNPQRSTUVXYZ[`abcdefhijklmpqr"; -static char table_a2b_base64[] = { +static const char table_a2b_base64[] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63, @@ -144,12 +144,12 @@ static char table_a2b_base64[] = { /* Max binary chunk size; limited only by available memory */ #define BASE64_MAXBIN ((PY_SSIZE_T_MAX - 3) / 2) -static unsigned char table_b2a_base64[] = +static const unsigned char table_b2a_base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static unsigned short crctab_hqx[256] = { +static const unsigned short crctab_hqx[256] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, @@ -977,7 +977,7 @@ binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) using byte-swap instructions. ********************************************************************/ -static unsigned int crc_32_tab[256] = { +static const unsigned int crc_32_tab[256] = { 0x00000000U, 0x77073096U, 0xee0e612cU, 0x990951baU, 0x076dc419U, 0x706af48fU, 0xe963a535U, 0x9e6495a3U, 0x0edb8832U, 0x79dcb8a4U, 0xe0d5e91eU, 0x97d2d988U, 0x09b64c2bU, 0x7eb17cbdU, 0xe7b82d07U, @@ -1201,7 +1201,7 @@ binascii_unhexlify_impl(PyModuleDef *module, Py_buffer *hexstr) return binascii_a2b_hex_impl(module, hexstr); } -static int table_hex[128] = { +static const int table_hex[128] = { -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index e2a2edf..33d7078 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -136,7 +136,7 @@ static int get_addr(const char *, int, struct addrinfo **, struct addrinfo *, int); static int str_isnumber(const char *); -static char *ai_errlist[] = { +static const char * const ai_errlist[] = { "success.", "address family for hostname not supported.", /* EAI_ADDRFAMILY */ "temporary failure in name resolution.", /* EAI_AGAIN */ diff --git a/Modules/getpath.c b/Modules/getpath.c index 03d292c..30a0e99 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -477,8 +477,8 @@ calculate_path(void) { extern wchar_t *Py_GetProgramName(void); - static wchar_t delimiter[2] = {DELIM, '\0'}; - static wchar_t separator[2] = {SEP, '\0'}; + static const wchar_t delimiter[2] = {DELIM, '\0'}; + static const wchar_t separator[2] = {SEP, '\0'}; char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ wchar_t *rtpypath = NULL; wchar_t *home = Py_GetPythonHome(); diff --git a/Modules/main.c b/Modules/main.c index 0fbdb69..4358cc8 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -42,11 +42,11 @@ static int orig_argc; #define PROGRAM_OPTS BASE_OPTS /* Short usage message (with %s for argv0) */ -static char *usage_line = +static const char usage_line[] = "usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; /* Long usage message, split into parts < 512 bytes */ -static char *usage_1 = "\ +static const char usage_1[] = "\ Options and arguments (and corresponding environment variables):\n\ -b : issue warnings about str(bytes_instance), str(bytearray_instance)\n\ and comparing bytes/bytearray with str. (-bb: issue errors)\n\ @@ -56,7 +56,7 @@ Options and arguments (and corresponding environment variables):\n\ -E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ -h : print this help message and exit (also --help)\n\ "; -static char *usage_2 = "\ +static const char usage_2[] = "\ -i : inspect interactively after running script; forces a prompt even\n\ if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ -I : isolate Python from the user's environment (implies -E and -s)\n\ @@ -67,7 +67,7 @@ static char *usage_2 = "\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ "; -static char *usage_3 = "\ +static const char usage_3[] = "\ -u : unbuffered binary stdout and stderr, stdin always buffered;\n\ also PYTHONUNBUFFERED=x\n\ see man page for details on internal buffering relating to '-u'\n\ @@ -79,7 +79,7 @@ static char *usage_3 = "\ -x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ -X opt : set implementation-specific option\n\ "; -static char *usage_4 = "\ +static const char usage_4[] = "\ file : program read from script file\n\ - : program read from stdin (default; interactive mode if a tty)\n\ arg ...: arguments passed to program in sys.argv[1:]\n\n\ @@ -88,14 +88,14 @@ PYTHONSTARTUP: file executed on interactive startup (no default)\n\ PYTHONPATH : '%c'-separated list of directories prefixed to the\n\ default module search path. The result is sys.path.\n\ "; -static char *usage_5 = +static const char usage_5[] = "PYTHONHOME : alternate directory (or %c).\n" " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" "PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n\ "; -static char *usage_6 = "\ +static const char usage_6[] = "\ PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ to seed the hashes of str, bytes and datetime objects. It can also be\n\ set to an integer in the range [0,4294967295] to get hash values with a\n\ diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 6471b8e..4c5f2ef 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -53,7 +53,7 @@ extern grammar _PyParser_Grammar; /* From graminit.c */ /* String constants used to initialize module attributes. * */ -static char parser_copyright_string[] = +static const char parser_copyright_string[] = "Copyright 1995-1996 by Virginia Polytechnic Institute & State\n\ University, Blacksburg, Virginia, USA, and Fred L. Drake, Jr., Reston,\n\ Virginia, USA. Portions copyright 1991-1995 by Stichting Mathematisch\n\ @@ -63,7 +63,7 @@ Centrum, Amsterdam, The Netherlands."; PyDoc_STRVAR(parser_doc_string, "This is an interface to Python's internal parser."); -static char parser_version_string[] = "0.5"; +static const char parser_version_string[] = "0.5"; typedef PyObject* (*SeqMaker) (Py_ssize_t length); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7ec1f47..7a2b661 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -9479,7 +9479,7 @@ os__getdiskusage_impl(PyModuleDef *module, Py_UNICODE *path) * sufficiently pervasive that it's not worth the loss of readability. */ struct constdef { - char *name; + const char *name; int value; }; @@ -10822,7 +10822,7 @@ os_getxattr_impl(PyModuleDef *module, path_t *path, path_t *attribute, for (i = 0; ; i++) { void *ptr; ssize_t result; - static Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; + static const Py_ssize_t buffer_sizes[] = {128, XATTR_SIZE_MAX, 0}; Py_ssize_t buffer_size = buffer_sizes[i]; if (!buffer_size) { path_error(path); @@ -10988,7 +10988,7 @@ os_listxattr_impl(PyModuleDef *module, path_t *path, int follow_symlinks) for (i = 0; ; i++) { char *start, *trace, *end; ssize_t length; - static Py_ssize_t buffer_sizes[] = { 256, XATTR_LIST_MAX, 0 }; + static const Py_ssize_t buffer_sizes[] = { 256, XATTR_LIST_MAX, 0 }; Py_ssize_t buffer_size = buffer_sizes[i]; if (!buffer_size) { /* ERANGE */ @@ -12821,7 +12821,7 @@ static struct PyModuleDef posixmodule = { }; -static char *have_functions[] = { +static const char * const have_functions[] = { #ifdef HAVE_FACCESSAT "HAVE_FACCESSAT", @@ -12956,7 +12956,7 @@ INITFUNC(void) { PyObject *m, *v; PyObject *list; - char **trace; + const char * const *trace; #if defined(HAVE_SYMLINK) && defined(MS_WINDOWS) win32_can_symlink = enable_symlink(); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index b3ac807..70f2db0 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1842,7 +1842,7 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) PyObject *pfd; static char *kwlist[] = {"ident", "filter", "flags", "fflags", "data", "udata", NULL}; - static char *fmt = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; + static const char fmt[] = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; EV_SET(&(self->e), 0, EVFILT_READ, EV_ADD, 0, 0, 0); /* defaults */ diff --git a/Modules/timemodule.c b/Modules/timemodule.c index d2caacd..31f0ce5 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -732,10 +732,10 @@ _asctime(struct tm *timeptr) { /* Inspired by Open Group reference implementation available at * http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html */ - static char wday_name[7][4] = { + static const char wday_name[7][4] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; - static char mon_name[12][4] = { + static const char mon_name[12][4] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index fe4e908..7d518fa 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -884,7 +884,7 @@ _gethash(const char *s, int len, int scale) return h; } -static char *hangul_syllables[][3] = { +static const char * const hangul_syllables[][3] = { { "G", "A", "" }, { "GG", "AE", "G" }, { "N", "YA", "GG" }, @@ -1057,7 +1057,7 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) int i, len1; *len = -1; for (i = 0; i < count; i++) { - char *s = hangul_syllables[i][column]; + const char *s = hangul_syllables[i][column]; len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int); if (len1 <= *len) continue; diff --git a/Objects/object.c b/Objects/object.c index e1718ea..417a97d 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -644,7 +644,7 @@ PyObject_Bytes(PyObject *v) /* Map rich comparison operators to their swapped version, e.g. LT <--> GT */ int _Py_SwappedOp[] = {Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE}; -static char *opstrings[] = {"<", "<=", "==", "!=", ">", ">="}; +static const char * const opstrings[] = {"<", "<=", "==", "!=", ">", ">="}; /* Perform a rich comparison, raising TypeError when the requested comparison operator is not supported. */ diff --git a/Objects/structseq.c b/Objects/structseq.c index 7209738..e315cba 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -4,9 +4,9 @@ #include "Python.h" #include "structmember.h" -static char visible_length_key[] = "n_sequence_fields"; -static char real_length_key[] = "n_fields"; -static char unnamed_fields_key[] = "n_unnamed_fields"; +static const char visible_length_key[] = "n_sequence_fields"; +static const char real_length_key[] = "n_fields"; +static const char unnamed_fields_key[] = "n_unnamed_fields"; /* Fields with this name have only a field index, not a field name. They are only allowed for indices < n_visible_fields. */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 341e18c..b4d23b2 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2690,7 +2690,7 @@ error: return NULL; } -static short slotoffsets[] = { +static const short slotoffsets[] = { -1, /* invalid slot */ #include "typeslots.inc" }; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a985d6f..fef184a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -272,7 +272,7 @@ raise_encode_exception(PyObject **exceptionObject, const char *reason); /* Same for linebreaks */ -static unsigned char ascii_linebreak[] = { +static const unsigned char ascii_linebreak[] = { 0, 0, 0, 0, 0, 0, 0, 0, /* 0x000A, * LINE FEED */ /* 0x000B, * LINE TABULATION */ @@ -4135,7 +4135,7 @@ unicode_decode_call_errorhandler_wchar( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, PyObject **output, Py_ssize_t *outpos) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -4243,7 +4243,7 @@ unicode_decode_call_errorhandler_writer( Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr, _PyUnicodeWriter *writer /* PyObject **output, Py_ssize_t *outpos */) { - static char *argparse = "O!n;decoding error handler must return (str, int) tuple"; + static const char *argparse = "O!n;decoding error handler must return (str, int) tuple"; PyObject *restuple = NULL; PyObject *repunicode = NULL; @@ -6560,7 +6560,7 @@ unicode_encode_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; + static const char *argparse = "On;encoding error handler must return (str/bytes, int) tuple"; Py_ssize_t len; PyObject *restuple; PyObject *resunicode; @@ -8572,7 +8572,7 @@ unicode_translate_call_errorhandler(const char *errors, Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos) { - static char *argparse = "O!n;translating error handler must return (str, int) tuple"; + static const char *argparse = "O!n;translating error handler must return (str, int) tuple"; Py_ssize_t i_newpos; PyObject *restuple; @@ -12156,7 +12156,7 @@ unicode_lower(PyObject *self) #define BOTHSTRIP 2 /* Arrays indexed by above */ -static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; +static const char * const stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; #define STRIPNAME(i) (stripformat[i]+3) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 629dee5..ebe9495 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -161,10 +161,10 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD #if 0 -static char with_msg[] = +static const char with_msg[] = "%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n"; -static char as_msg[] = +static const char as_msg[] = "%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n"; static void diff --git a/Parser/pgen.c b/Parser/pgen.c index f3031ae..6ecb311 100644 --- a/Parser/pgen.c +++ b/Parser/pgen.c @@ -134,7 +134,7 @@ addnfa(nfagrammar *gr, char *name) #ifdef Py_DEBUG -static char REQNFMT[] = "metacompile: less than %d children\n"; +static const char REQNFMT[] = "metacompile: less than %d children\n"; #define REQN(i, count) do { \ if (i < count) { \ diff --git a/Python/ast.c b/Python/ast.c index 77ebc83..328ee5d 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -870,7 +870,7 @@ get_operator(const node *n) } } -static const char* FORBIDDEN[] = { +static const char * const FORBIDDEN[] = { "None", "True", "False", @@ -887,7 +887,7 @@ forbidden_name(struct compiling *c, identifier name, const node *n, return 1; } if (full_checks) { - const char **p; + const char * const *p; for (p = FORBIDDEN; *p; p++) { if (PyUnicode_CompareWithASCIIString(name, *p) == 0) { ast_error(c, n, "assignment to keyword"); diff --git a/Python/dtoa.c b/Python/dtoa.c index 3da546e..3121cd6 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -747,7 +747,7 @@ pow5mult(Bigint *b, int k) { Bigint *b1, *p5, *p51; int i; - static int p05[3] = { 5, 25, 125 }; + static const int p05[3] = { 5, 25, 125 }; if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); @@ -803,7 +803,7 @@ pow5mult(Bigint *b, int k) { Bigint *b1, *p5, *p51; int i; - static int p05[3] = { 5, 25, 125 }; + static const int p05[3] = { 5, 25, 125 }; if ((i = k & 3)) { b = multadd(b, p05[i-1], 0); diff --git a/Python/formatter_unicode.c b/Python/formatter_unicode.c index 056bb76..a428fbe 100644 --- a/Python/formatter_unicode.c +++ b/Python/formatter_unicode.c @@ -656,7 +656,7 @@ fill_number(_PyUnicodeWriter *writer, const NumberFieldWidths *spec, return 0; } -static char no_grouping[1] = {CHAR_MAX}; +static const char no_grouping[1] = {CHAR_MAX}; /* Find the decimal point character(s?), thousands_separator(s?), and grouping description, either for the current locale if type is diff --git a/Python/import.c b/Python/import.c index edf030d..8d7bfe9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -320,7 +320,7 @@ PyImport_GetModuleDict(void) /* List of names to clear in sys */ -static char* sys_deletes[] = { +static const char * const sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", @@ -330,7 +330,7 @@ static char* sys_deletes[] = { NULL }; -static char* sys_files[] = { +static const char * const sys_files[] = { "stdin", "__stdin__", "stdout", "__stdout__", "stderr", "__stderr__", @@ -347,7 +347,7 @@ PyImport_Cleanup(void) PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; PyObject *weaklist = NULL; - char **p; + const char * const *p; if (modules == NULL) return; /* Already done */ diff --git a/Python/importdl.c b/Python/importdl.c index 1aa585d..ac03289 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -23,8 +23,8 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif -static const char *ascii_only_prefix = "PyInit"; -static const char *nonascii_prefix = "PyInitU"; +static const char * const ascii_only_prefix = "PyInit"; +static const char * const nonascii_prefix = "PyInitU"; /* Get the variable part of a module's export symbol name. * Returns a bytes instance. For non-ASCII-named modules, the name is diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c index 98429d4..a85790e 100644 --- a/Python/mystrtoul.c +++ b/Python/mystrtoul.c @@ -17,7 +17,7 @@ * smallmax[base] is the largest unsigned long i such that * i * base doesn't overflow unsigned long. */ -static unsigned long smallmax[] = { +static const unsigned long smallmax[] = { 0, /* bases 0 and 1 are invalid */ 0, ULONG_MAX / 2, @@ -62,14 +62,14 @@ static unsigned long smallmax[] = { * Note that this is pessimistic if sizeof(long) > 4. */ #if SIZEOF_LONG == 4 -static int digitlimit[] = { +static const int digitlimit[] = { 0, 0, 32, 20, 16, 13, 12, 11, 10, 10, /* 0 - 9 */ 9, 9, 8, 8, 8, 8, 8, 7, 7, 7, /* 10 - 19 */ 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, /* 20 - 29 */ 6, 6, 6, 6, 6, 6, 6}; /* 30 - 36 */ #elif SIZEOF_LONG == 8 /* [int(math.floor(math.log(2**64, i))) for i in range(2, 37)] */ -static int digitlimit[] = { +static const int digitlimit[] = { 0, 0, 64, 40, 32, 27, 24, 22, 21, 20, /* 0 - 9 */ 19, 18, 17, 17, 16, 16, 16, 15, 15, 15, /* 10 - 19 */ 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, /* 20 - 29 */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 209c908..5f3af92 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -881,12 +881,12 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, #define OFS_E 2 /* The lengths of these are known to the code below, so don't change them */ -static char *lc_float_strings[] = { +static const char * const lc_float_strings[] = { "inf", "nan", "e", }; -static char *uc_float_strings[] = { +static const char * const uc_float_strings[] = { "INF", "NAN", "E", @@ -925,7 +925,8 @@ static char * format_float_short(double d, char format_code, int mode, int precision, int always_add_sign, int add_dot_0_if_integer, - int use_alt_formatting, char **float_strings, int *type) + int use_alt_formatting, const char * const *float_strings, + int *type) { char *buf = NULL; char *p = NULL; @@ -1176,7 +1177,7 @@ PyAPI_FUNC(char *) PyOS_double_to_string(double val, int flags, int *type) { - char **float_strings = lc_float_strings; + const char * const *float_strings = lc_float_strings; int mode; /* Validate format_code, and map upper and lower case. Compute the diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 1a5dab5..cfe197b 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -785,11 +785,11 @@ print_exception(PyObject *f, PyObject *value) PyErr_Clear(); } -static const char *cause_message = +static const char cause_message[] = "\nThe above exception was the direct cause " "of the following exception:\n\n"; -static const char *context_message = +static const char context_message[] = "\nDuring handling of the above exception, " "another exception occurred:\n\n"; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index e0aa233..f784f75 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -346,8 +346,10 @@ static PyObject *whatstrings[7] = {NULL, NULL, NULL, NULL, NULL, NULL, NULL}; static int trace_init(void) { - static char *whatnames[7] = {"call", "exception", "line", "return", - "c_call", "c_exception", "c_return"}; + static const char * const whatnames[7] = { + "call", "exception", "line", "return", + "c_call", "c_exception", "c_return" + }; PyObject *name; int i; for (i = 0; i < 7; ++i) { -- cgit v0.12 From ef1585eb9a488ae8ce3ff057f43a7048b941cc1c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 25 Dec 2015 20:01:53 +0200 Subject: Issue #25923: Added more const qualifiers to signatures of static and private functions. --- Include/bytes_methods.h | 6 ++--- Include/pythonrun.h | 8 +++---- Misc/coverity_model.c | 2 +- Modules/_ctypes/_ctypes.c | 4 ++-- Modules/_ctypes/callbacks.c | 2 +- Modules/_ctypes/callproc.c | 2 +- Modules/_ctypes/ctypes.h | 2 +- Modules/_curses_panel.c | 2 +- Modules/_datetimemodule.c | 2 +- Modules/_io/_iomodule.h | 2 +- Modules/_io/bufferedio.c | 2 +- Modules/_io/fileio.c | 4 ++-- Modules/_io/textio.c | 20 ++++++++-------- Modules/_json.c | 4 ++-- Modules/_localemodule.c | 2 +- Modules/_pickle.c | 2 +- Modules/_posixsubprocess.c | 2 +- Modules/_scproxy.c | 2 +- Modules/_sre.c | 2 +- Modules/_ssl.c | 4 ++-- Modules/_struct.c | 6 ++--- Modules/_testmultiphase.c | 2 +- Modules/_tkinter.c | 8 +++---- Modules/_winapi.c | 4 ++-- Modules/binascii.c | 44 +++++++++++++++++++++------------- Modules/gcmodule.c | 2 +- Modules/getaddrinfo.c | 2 +- Modules/main.c | 2 +- Modules/mathmodule.c | 4 ++-- Modules/parsermodule.c | 10 ++++---- Modules/posixmodule.c | 43 +++++++++++++++++++-------------- Modules/pyexpat.c | 6 ++--- Modules/socketmodule.c | 4 ++-- Modules/timemodule.c | 2 +- Modules/xxlimited.c | 2 +- Modules/xxmodule.c | 2 +- Modules/zipimport.c | 2 +- Modules/zlibmodule.c | 2 +- Objects/bytearrayobject.c | 8 +++---- Objects/bytes_methods.c | 6 ++--- Objects/bytesobject.c | 2 +- Objects/descrobject.c | 2 +- Objects/dictobject.c | 2 +- Objects/memoryobject.c | 4 ++-- Objects/typeobject.c | 10 ++++---- Objects/unicodeobject.c | 6 ++--- Objects/weakrefobject.c | 2 +- Parser/pgen.c | 4 ++-- Parser/pgenmain.c | 4 ++-- Parser/tokenizer.c | 6 ++--- Python/_warnings.c | 2 +- Python/ceval.c | 4 ++-- Python/dtoa.c | 2 +- Python/dynload_win.c | 2 +- Python/getargs.c | 58 ++++++++++++++++++++++----------------------- Python/marshal.c | 34 +++++++++++++------------- Python/modsupport.c | 4 ++-- Python/pylifecycle.c | 4 ++-- Python/pythonrun.c | 8 +++---- Python/symtable.c | 4 ++-- 60 files changed, 210 insertions(+), 191 deletions(-) diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h index 11d5f42..dbc033c 100644 --- a/Include/bytes_methods.h +++ b/Include/bytes_methods.h @@ -17,9 +17,9 @@ extern PyObject* _Py_bytes_istitle(const char *cptr, Py_ssize_t len); /* These store their len sized answer in the given preallocated *result arg. */ extern void _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len); extern void _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len); -extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len); -extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len); -extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_title(char *result, const char *s, Py_ssize_t len); +extern void _Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len); +extern void _Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len); /* The maketrans() static method. */ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); diff --git a/Include/pythonrun.h b/Include/pythonrun.h index f92148d..3b93a80 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -66,8 +66,8 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( const char *filename, /* decoded from the filesystem encoding */ const char* enc, int start, - char *ps1, - char *ps2, + const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); @@ -76,8 +76,8 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( PyObject *filename, const char* enc, int start, - char *ps1, - char *ps2, + const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); diff --git a/Misc/coverity_model.c b/Misc/coverity_model.c index 493e7c1..f776d76 100644 --- a/Misc/coverity_model.c +++ b/Misc/coverity_model.c @@ -94,7 +94,7 @@ wchar_t *Py_DecodeLocale(const char* arg, size_t *size) } /* Parser/pgenmain.c */ -grammar *getgrammar(char *filename) +grammar *getgrammar(const char *filename) { grammar *g; __coverity_tainted_data_sink__(filename); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fccc8ba..ac5e5fa 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3195,7 +3195,7 @@ _validate_paramflags(PyTypeObject *type, PyObject *paramflags) } static int -_get_name(PyObject *obj, char **pname) +_get_name(PyObject *obj, const char **pname) { #ifdef MS_WIN32 if (PyLong_Check(obj)) { @@ -3223,7 +3223,7 @@ _get_name(PyObject *obj, char **pname) static PyObject * PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) { - char *name; + const char *name; int (* address)(void); PyObject *ftuple; PyObject *dll; diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 7cd6164..00e8e66 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -77,7 +77,7 @@ PyTypeObject PyCThunk_Type = { /**************************************************************/ static void -PrintError(char *msg, ...) +PrintError(const char *msg, ...) { char buf[512]; PyObject *f = PySys_GetObject("stderr"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 68981fe..870a0d4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -928,7 +928,7 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) * Raise a new exception 'exc_class', adding additional text to the original * exception string. */ -void _ctypes_extend_error(PyObject *exc_class, char *fmt, ...) +void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 0d3f724..b06ba8a 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -327,7 +327,7 @@ extern int PyCData_set(PyObject *dst, PyObject *type, SETFUNC setfunc, PyObject *value, Py_ssize_t index, Py_ssize_t size, char *ptr); -extern void _ctypes_extend_error(PyObject *exc_class, char *fmt, ...); +extern void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...); struct basespec { CDataObject *base; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index a9c406f..26bf094 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -56,7 +56,7 @@ static struct PyModuleDef _curses_panelmodule; */ static PyObject * -PyCursesCheckERR(int code, char *fname) +PyCursesCheckERR(int code, const char *fname) { if (code != ERR) { Py_INCREF(Py_None); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index e8b7ae8..3e5e195 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -873,7 +873,7 @@ get_tzinfo_member(PyObject *self) * this returns NULL. Else result is returned. */ static PyObject * -call_tzinfo_method(PyObject *tzinfo, char *name, PyObject *tzinfoarg) +call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) { PyObject *offset; diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 0c6eae2..3c48ff3 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -60,7 +60,7 @@ extern PyObject *_PyIncrementalNewlineDecoder_decode( * Otherwise, the line ending is specified by readnl, a str object */ extern Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - int kind, char *start, char *end, Py_ssize_t *consumed); + int kind, const char *start, const char *end, Py_ssize_t *consumed); /* Return 1 if an EnvironmentError with errno == EINTR is set (and then clears the error indicator), 0 otherwise. diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6bb2200..16f8cdf 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -659,7 +659,7 @@ _bufferedreader_raw_read(buffered *self, char *start, Py_ssize_t len); /* Sets the current error to BlockingIOError */ static void -_set_BlockingIOError(char *msg, Py_ssize_t written) +_set_BlockingIOError(const char *msg, Py_ssize_t written) { PyObject *err; PyErr_Clear(); diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index dbd604a..8bf3922 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -540,7 +540,7 @@ err_closed(void) } static PyObject * -err_mode(char *action) +err_mode(const char *action) { _PyIO_State *state = IO_STATE(); if (state != NULL) @@ -1043,7 +1043,7 @@ _io_FileIO_truncate_impl(fileio *self, PyObject *posobj) } #endif /* HAVE_FTRUNCATE */ -static char * +static const char * mode_string(fileio *self) { if (self->created) { diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index d018623..140da10 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1648,8 +1648,8 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) /* NOTE: `end` must point to the real end of the Py_UCS4 storage, that is to the NUL character. Otherwise the function will produce incorrect results. */ -static char * -find_control_char(int kind, char *s, char *end, Py_UCS4 ch) +static const char * +find_control_char(int kind, const char *s, const char *end, Py_UCS4 ch) { if (kind == PyUnicode_1BYTE_KIND) { assert(ch < 256); @@ -1669,13 +1669,13 @@ find_control_char(int kind, char *s, char *end, Py_UCS4 ch) Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - int kind, char *start, char *end, Py_ssize_t *consumed) + int kind, const char *start, const char *end, Py_ssize_t *consumed) { Py_ssize_t len = ((char*)end - (char*)start)/kind; if (translated) { /* Newlines are already translated, only search for \n */ - char *pos = find_control_char(kind, start, end, '\n'); + const char *pos = find_control_char(kind, start, end, '\n'); if (pos != NULL) return (pos - start)/kind + 1; else { @@ -1687,7 +1687,7 @@ _PyIO_find_line_ending( /* Universal newline search. Find any of \r, \r\n, \n * The decoder ensures that \r\n are not split in two pieces */ - char *s = start; + const char *s = start; for (;;) { Py_UCS4 ch; /* Fast path for non-control chars. The loop always ends @@ -1717,21 +1717,21 @@ _PyIO_find_line_ending( /* Assume that readnl is an ASCII character. */ assert(PyUnicode_KIND(readnl) == PyUnicode_1BYTE_KIND); if (readnl_len == 1) { - char *pos = find_control_char(kind, start, end, nl[0]); + const char *pos = find_control_char(kind, start, end, nl[0]); if (pos != NULL) return (pos - start)/kind + 1; *consumed = len; return -1; } else { - char *s = start; - char *e = end - (readnl_len - 1)*kind; - char *pos; + const char *s = start; + const char *e = end - (readnl_len - 1)*kind; + const char *pos; if (e < s) e = s; while (s < e) { Py_ssize_t i; - char *pos = find_control_char(kind, s, end, nl[0]); + const char *pos = find_control_char(kind, s, end, nl[0]); if (pos == NULL || pos >= e) break; for (i = 1; i < readnl_len; i++) { diff --git a/Modules/_json.c b/Modules/_json.c index f63d758..3c857ae 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -112,7 +112,7 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, PyObject *dct, Py_ssiz static PyObject * _encoded_const(PyObject *obj); static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); +raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end); static PyObject * encoder_encode_string(PyEncoderObject *s, PyObject *obj); static PyObject * @@ -323,7 +323,7 @@ escape_unicode(PyObject *pystr) } static void -raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) +raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) { /* Use JSONDecodeError exception to raise a nice looking ValueError subclass */ static PyObject *JSONDecodeError = NULL; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index b1d6add..a92fb10 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -49,7 +49,7 @@ PyDoc_STRVAR(setlocale__doc__, /* the grouping is terminated by either 0 or CHAR_MAX */ static PyObject* -copy_grouping(char* s) +copy_grouping(const char* s) { int i; PyObject *result, *val = NULL; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 487b533..9d36346 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2187,7 +2187,7 @@ error: } static int -write_utf8(PicklerObject *self, char *data, Py_ssize_t size) +write_utf8(PicklerObject *self, const char *data, Py_ssize_t size) { char header[9]; Py_ssize_t len; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 8bedab5..a0109fb 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -72,7 +72,7 @@ _enable_gc(int need_to_reenable_gc, PyObject *gc_module) /* Convert ASCII to a positive int, no libc call. no overflow. -1 on error. */ static int -_pos_int_from_ascii(char *name) +_pos_int_from_ascii(const char *name) { int num = 0; while (*name >= '0' && *name <= '9') { diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 66b6e34..68be458 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -130,7 +130,7 @@ error: } static int -set_proxy(PyObject* proxies, char* proto, CFDictionaryRef proxyDict, +set_proxy(PyObject* proxies, const char* proto, CFDictionaryRef proxyDict, CFStringRef enabledKey, CFStringRef hostKey, CFStringRef portKey) { diff --git a/Modules/_sre.c b/Modules/_sre.c index f597a70..fb0ab03 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -714,7 +714,7 @@ _sre_SRE_Pattern_search_impl(PatternObject *self, PyObject *string, } static PyObject* -call(char* module, char* function, PyObject* args) +call(const char* module, const char* function, PyObject* args) { PyObject* name; PyObject* mod; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8818d26..5968ed5 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -378,7 +378,7 @@ fail: } static PyObject * -PySSL_SetError(PySSLSocket *obj, int ret, char *filename, int lineno) +PySSL_SetError(PySSLSocket *obj, int ret, const char *filename, int lineno) { PyObject *type = PySSLErrorObject; char *errstr = NULL; @@ -460,7 +460,7 @@ PySSL_SetError(PySSLSocket *obj, int ret, char *filename, int lineno) } static PyObject * -_setSSLError (char *errstr, int errcode, char *filename, int lineno) { +_setSSLError (const char *errstr, int errcode, const char *filename, int lineno) { if (errstr == NULL) errcode = ERR_peek_last_error(); diff --git a/Modules/_struct.c b/Modules/_struct.c index b18c71d..820e004 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1189,7 +1189,7 @@ static formatdef lilendian_table[] = { static const formatdef * -whichtable(char **pfmt) +whichtable(const char **pfmt) { const char *fmt = (*pfmt)++; /* May be backed out of later */ switch (*fmt) { @@ -1268,7 +1268,7 @@ prepare_s(PyStructObject *self) fmt = PyBytes_AS_STRING(self->s_format); - f = whichtable((char **)&fmt); + f = whichtable(&fmt); s = fmt; size = 0; @@ -1457,7 +1457,7 @@ s_dealloc(PyStructObject *s) } static PyObject * -s_unpack_internal(PyStructObject *soself, char *startfrom) { +s_unpack_internal(PyStructObject *soself, const char *startfrom) { formatcode *code; Py_ssize_t i = 0; PyObject *result = PyTuple_New(soself->s_len); diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 2005205..03eda27 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -61,7 +61,7 @@ Example_getattro(ExampleObject *self, PyObject *name) } static int -Example_setattr(ExampleObject *self, char *name, PyObject *v) +Example_setattr(ExampleObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 41ad5f9..768df81 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -841,7 +841,7 @@ PyTclObject_dealloc(PyTclObject *self) Py_DECREF(tp); } -static char* +static const char * PyTclObject_TclString(PyObject *self) { return Tcl_GetString(((PyTclObject*)self)->value); @@ -1726,7 +1726,7 @@ static int varname_converter(PyObject *in, void *_out) { char *s; - char **out = (char**)_out; + const char **out = (const char**)_out; if (PyBytes_Check(in)) { if (PyBytes_Size(in) > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "bytes object is too long"); @@ -1846,7 +1846,7 @@ var_invoke(EventFunc func, PyObject *selfptr, PyObject *args, int flags) static PyObject * SetVar(PyObject *self, PyObject *args, int flags) { - char *name1, *name2; + const char *name1, *name2; PyObject *newValue; PyObject *res = NULL; Tcl_Obj *newval, *ok; @@ -1915,7 +1915,7 @@ Tkapp_GlobalSetVar(PyObject *self, PyObject *args) static PyObject * GetVar(PyObject *self, PyObject *args, int flags) { - char *name1, *name2=NULL; + const char *name1, *name2=NULL; PyObject *res = NULL; Tcl_Obj *tres; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 3e7f187..c4d4264 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -675,7 +675,7 @@ _winapi_CreatePipe_impl(PyModuleDef *module, PyObject *pipe_attrs, /* helpers for createprocess */ static unsigned long -getulong(PyObject* obj, char* name) +getulong(PyObject* obj, const char* name) { PyObject* value; unsigned long ret; @@ -691,7 +691,7 @@ getulong(PyObject* obj, char* name) } static HANDLE -gethandle(PyObject* obj, char* name) +gethandle(PyObject* obj, const char* name) { PyObject* value; HANDLE ret; diff --git a/Modules/binascii.c b/Modules/binascii.c index 9df48da..a306acd 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -256,7 +256,8 @@ static PyObject * binascii_a2b_uu_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=5779f39b0b48459f input=7cafeaf73df63d1c]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -342,7 +343,8 @@ static PyObject * binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=181021b69bb9a414 input=00fdf458ce8b465b]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -389,7 +391,7 @@ binascii_b2a_uu_impl(PyModuleDef *module, Py_buffer *data) static int -binascii_find_valid(unsigned char *s, Py_ssize_t slen, int num) +binascii_find_valid(const unsigned char *s, Py_ssize_t slen, int num) { /* Finds & returns the (num+1)th ** valid character for base64, or -1 if none. @@ -426,7 +428,8 @@ static PyObject * binascii_a2b_base64_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=3e351b702bed56d2 input=5872acf6e1cac243]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -522,7 +525,8 @@ static PyObject * binascii_b2a_base64_impl(PyModuleDef *module, Py_buffer *data, int newline) /*[clinic end generated code: output=19e1dd719a890b50 input=7b2ea6fa38d8924c]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -589,7 +593,8 @@ static PyObject * binascii_a2b_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=60bcdbbd28b105cd input=0d914c680e0eed55]*/ { - unsigned char *ascii_data, *bin_data; + const unsigned char *ascii_data; + unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -667,7 +672,8 @@ static PyObject * binascii_rlecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=0905da344dbf0648 input=e1f1712447a82b09]*/ { - unsigned char *in_data, *out_data; + const unsigned char *in_data; + unsigned char *out_data; unsigned char ch; Py_ssize_t in, inend, len; _PyBytesWriter writer; @@ -728,7 +734,8 @@ static PyObject * binascii_b2a_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=5a987810d5e3cdbb input=9596ebe019fe12ba]*/ { - unsigned char *ascii_data, *bin_data; + unsigned char *ascii_data; + const unsigned char *bin_data; int leftbits = 0; unsigned char this_ch; unsigned int leftchar = 0; @@ -782,7 +789,8 @@ static PyObject * binascii_rledecode_hqx_impl(PyModuleDef *module, Py_buffer *data) /*[clinic end generated code: output=f7afd89b789946ab input=54cdd49fc014402c]*/ { - unsigned char *in_data, *out_data; + const unsigned char *in_data; + unsigned char *out_data; unsigned char in_byte, in_repeat; Py_ssize_t in_len; _PyBytesWriter writer; @@ -899,7 +907,7 @@ static unsigned int binascii_crc_hqx_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) /*[clinic end generated code: output=167c2dac62625717 input=add8c53712ccceda]*/ { - unsigned char *bin_data; + const unsigned char *bin_data; Py_ssize_t len; crc &= 0xffff; @@ -1050,7 +1058,7 @@ binascii_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) #ifdef USE_ZLIB_CRC32 /* This was taken from zlibmodule.c PyZlib_crc32 (but is PY_SSIZE_T_CLEAN) */ { - Byte *buf; + const Byte *buf; Py_ssize_t len; int signed_val; @@ -1061,7 +1069,7 @@ binascii_crc32_impl(PyModuleDef *module, Py_buffer *data, unsigned int crc) } #else /* USE_ZLIB_CRC32 */ { /* By Jim Ahlstrom; All rights transferred to CNRI */ - unsigned char *bin_data; + const unsigned char *bin_data; Py_ssize_t len; unsigned int result; @@ -1144,7 +1152,7 @@ static PyObject * binascii_a2b_hex_impl(PyModuleDef *module, Py_buffer *hexstr) /*[clinic end generated code: output=d61da452b5c6d290 input=9e1e7f2f94db24fd]*/ { - char* argbuf; + const char* argbuf; Py_ssize_t arglen; PyObject *retval; char* retbuf; @@ -1232,7 +1240,8 @@ binascii_a2b_qp_impl(PyModuleDef *module, Py_buffer *data, int header) { Py_ssize_t in, out; char ch; - unsigned char *ascii_data, *odata; + const unsigned char *ascii_data; + unsigned char *odata; Py_ssize_t datalen = 0; PyObject *rv; @@ -1338,13 +1347,14 @@ binascii_b2a_qp_impl(PyModuleDef *module, Py_buffer *data, int quotetabs, /*[clinic end generated code: output=a87ca9ccb94e2a9f input=7f2a9aaa008e92b2]*/ { Py_ssize_t in, out; - unsigned char *databuf, *odata; + const unsigned char *databuf; + unsigned char *odata; Py_ssize_t datalen = 0, odatalen = 0; PyObject *rv; unsigned int linelen = 0; unsigned char ch; int crlf = 0; - unsigned char *p; + const unsigned char *p; databuf = data->buf; datalen = data->len; @@ -1353,7 +1363,7 @@ binascii_b2a_qp_impl(PyModuleDef *module, Py_buffer *data, int quotetabs, /* XXX: this function has the side effect of converting all of * the end of lines to be the same depending on this detection * here */ - p = (unsigned char *) memchr(databuf, '\n', datalen); + p = (const unsigned char *) memchr(databuf, '\n', datalen); if ((p != NULL) && (p > databuf) && (*(p-1) == '\r')) crlf = 1; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index cb7222d..0c6f444 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -738,7 +738,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) } static void -debug_cycle(char *msg, PyObject *op) +debug_cycle(const char *msg, PyObject *op) { PySys_FormatStderr("gc: %s <%s %p>\n", msg, Py_TYPE(op)->tp_name, op); diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index 33d7078..13a9e40 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -198,7 +198,7 @@ if (pai->ai_flags & AI_CANONNAME) {\ #define ERR(err) { error = (err); goto bad; } -char * +const char * gai_strerror(int ecode) { if (ecode < 0 || ecode > EAI_MAX) diff --git a/Modules/main.c b/Modules/main.c index 4358cc8..35d07fd 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -103,7 +103,7 @@ PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ "; static int -usage(int exitcode, wchar_t* program) +usage(int exitcode, const wchar_t* program) { FILE *f = exitcode ? stderr : stdout; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 9359eb2..6b3e139 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -876,7 +876,7 @@ math_1_to_int(PyObject *arg, double (*func) (double), int can_overflow) } static PyObject * -math_2(PyObject *args, double (*func) (double, double), char *funcname) +math_2(PyObject *args, double (*func) (double, double), const char *funcname) { PyObject *ox, *oy; double x, y, r; @@ -1673,7 +1673,7 @@ PyDoc_STRVAR(math_modf_doc, in that int is larger than PY_SSIZE_T_MAX. */ static PyObject* -loghelper(PyObject* arg, double (*func)(double), char *funcname) +loghelper(PyObject* arg, double (*func)(double), const char *funcname) { /* If it is int, do it ourselves. */ if (PyLong_Check(arg)) { diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 4c5f2ef..deae049 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -578,13 +578,13 @@ parser_issuite(PyST_Object *self, PyObject *args, PyObject *kw) } -/* err_string(char* message) +/* err_string(const char* message) * * Sets the error string for an exception of type ParserError. * */ static void -err_string(char *message) +err_string(const char *message) { PyErr_SetString(parser_error, message); } @@ -597,7 +597,7 @@ err_string(char *message) * */ static PyObject* -parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) +parser_do_parse(PyObject *args, PyObject *kw, const char *argspec, int type) { char* string = 0; PyObject* res = 0; @@ -984,7 +984,7 @@ build_node_tree(PyObject *tuple) /* * Validation routines used within the validation section: */ -static int validate_terminal(node *terminal, int type, char *string); +static int validate_terminal(node *terminal, int type, const char *string); #define validate_ampersand(ch) validate_terminal(ch, AMPER, "&") #define validate_circumflex(ch) validate_terminal(ch, CIRCUMFLEX, "^") @@ -1082,7 +1082,7 @@ validate_numnodes(node *n, int num, const char *const name) static int -validate_terminal(node *terminal, int type, char *string) +validate_terminal(node *terminal, int type, const char *string) { int res = (validate_ntype(terminal, type) && ((string == 0) || (strcmp(string, STR(terminal)) == 0))); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7a2b661..367e4f2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -949,7 +949,8 @@ path_converter(PyObject *o, void *p) { } static void -argument_unavailable_error(char *function_name, char *argument_name) { +argument_unavailable_error(const char *function_name, const char *argument_name) +{ PyErr_Format(PyExc_NotImplementedError, "%s%s%s unavailable on this platform", (function_name != NULL) ? function_name : "", @@ -972,7 +973,8 @@ dir_fd_unavailable(PyObject *o, void *p) } static int -fd_specified(char *function_name, int fd) { +fd_specified(const char *function_name, int fd) +{ if (fd == -1) return 0; @@ -981,7 +983,8 @@ fd_specified(char *function_name, int fd) { } static int -follow_symlinks_specified(char *function_name, int follow_symlinks) { +follow_symlinks_specified(const char *function_name, int follow_symlinks) +{ if (follow_symlinks) return 0; @@ -990,7 +993,8 @@ follow_symlinks_specified(char *function_name, int follow_symlinks) { } static int -path_and_dir_fd_invalid(char *function_name, path_t *path, int dir_fd) { +path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) +{ if (!path->narrow && !path->wide && (dir_fd != DEFAULT_DIR_FD)) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", @@ -1001,7 +1005,8 @@ path_and_dir_fd_invalid(char *function_name, path_t *path, int dir_fd) { } static int -dir_fd_and_fd_invalid(char *function_name, int dir_fd, int fd) { +dir_fd_and_fd_invalid(const char *function_name, int dir_fd, int fd) +{ if ((dir_fd != DEFAULT_DIR_FD) && (fd != -1)) { PyErr_Format(PyExc_ValueError, "%s: can't specify both dir_fd and fd", @@ -1012,8 +1017,9 @@ dir_fd_and_fd_invalid(char *function_name, int dir_fd, int fd) { } static int -fd_and_follow_symlinks_invalid(char *function_name, int fd, - int follow_symlinks) { +fd_and_follow_symlinks_invalid(const char *function_name, int fd, + int follow_symlinks) +{ if ((fd > 0) && (!follow_symlinks)) { PyErr_Format(PyExc_ValueError, "%s: cannot use fd and follow_symlinks together", @@ -1024,8 +1030,9 @@ fd_and_follow_symlinks_invalid(char *function_name, int fd, } static int -dir_fd_and_follow_symlinks_invalid(char *function_name, int dir_fd, - int follow_symlinks) { +dir_fd_and_follow_symlinks_invalid(const char *function_name, int dir_fd, + int follow_symlinks) +{ if ((dir_fd != DEFAULT_DIR_FD) && (!follow_symlinks)) { PyErr_Format(PyExc_ValueError, "%s: cannot use dir_fd and follow_symlinks together", @@ -1220,7 +1227,7 @@ posix_error(void) #ifdef MS_WINDOWS static PyObject * -win32_error(char* function, const char* filename) +win32_error(const char* function, const char* filename) { /* XXX We should pass the function name along in the future. (winreg.c also wants to pass the function name.) @@ -1235,7 +1242,7 @@ win32_error(char* function, const char* filename) } static PyObject * -win32_error_object(char* function, PyObject* filename) +win32_error_object(const char* function, PyObject* filename) { /* XXX - see win32_error for comments on 'function' */ errno = GetLastError(); @@ -2100,7 +2107,7 @@ _pystat_fromstructstat(STRUCT_STAT *st) static PyObject * -posix_do_stat(char *function_name, path_t *path, +posix_do_stat(const char *function_name, path_t *path, int dir_fd, int follow_symlinks) { STRUCT_STAT st; @@ -4561,7 +4568,7 @@ typedef struct { #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) static int -utime_dir_fd(utime_t *ut, int dir_fd, char *path, int follow_symlinks) +utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { #ifdef HAVE_UTIMENSAT int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; @@ -4610,7 +4617,7 @@ utime_fd(utime_t *ut, int fd) #ifdef UTIME_HAVE_NOFOLLOW_SYMLINKS static int -utime_nofollow_symlinks(utime_t *ut, char *path) +utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT UTIME_TO_TIMESPEC; @@ -4626,7 +4633,7 @@ utime_nofollow_symlinks(utime_t *ut, char *path) #ifndef MS_WINDOWS static int -utime_default(utime_t *ut, char *path) +utime_default(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT UTIME_TO_TIMESPEC; @@ -7323,7 +7330,7 @@ _check_dirW(WCHAR *src, WCHAR *dest) /* Return True if the path at src relative to dest is a directory */ static int -_check_dirA(char *src, char *dest) +_check_dirA(const char *src, char *dest) { WIN32_FILE_ATTRIBUTE_DATA src_info; char dest_parent[MAX_PATH]; @@ -11835,7 +11842,7 @@ error: #else /* POSIX */ static char * -join_path_filename(char *path_narrow, char* filename, Py_ssize_t filename_len) +join_path_filename(const char *path_narrow, const char* filename, Py_ssize_t filename_len) { Py_ssize_t path_len; Py_ssize_t size; @@ -11867,7 +11874,7 @@ join_path_filename(char *path_narrow, char* filename, Py_ssize_t filename_len) } static PyObject * -DirEntry_from_posix_info(path_t *path, char *name, Py_ssize_t name_len, +DirEntry_from_posix_info(path_t *path, const char *name, Py_ssize_t name_len, ino_t d_ino #ifdef HAVE_DIRENT_D_TYPE , unsigned char d_type diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index b45e3da..9267c69 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -91,7 +91,7 @@ static struct HandlerInfo handler_info[64]; * false on an exception. */ static int -set_error_attr(PyObject *err, char *name, int value) +set_error_attr(PyObject *err, const char *name, int value) { PyObject *v = PyLong_FromLong(value); @@ -218,7 +218,7 @@ flag_error(xmlparseobject *self) } static PyObject* -call_with_frame(char *funcname, int lineno, PyObject* func, PyObject* args, +call_with_frame(const char *funcname, int lineno, PyObject* func, PyObject* args, xmlparseobject *self) { PyObject *res; @@ -766,7 +766,7 @@ readinst(char *buf, int buf_size, PyObject *meth) { PyObject *str; Py_ssize_t len; - char *ptr; + const char *ptr; str = PyObject_CallFunction(meth, "n", buf_size); if (str == NULL) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bae9634..7ab534e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -904,7 +904,7 @@ static PyThread_type_lock netdb_lock; an error occurred; then an exception is raised. */ static int -setipaddr(char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) +setipaddr(const char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) { struct addrinfo hints, *res; int error; @@ -1085,7 +1085,7 @@ makeipaddr(struct sockaddr *addr, int addrlen) an error occurred. */ static int -setbdaddr(char *name, bdaddr_t *bdaddr) +setbdaddr(const char *name, bdaddr_t *bdaddr) { unsigned int b0, b1, b2, b3, b4, b5; char ch; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 31f0ce5..0b6d461 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -311,7 +311,7 @@ tmtotuple(struct tm *p) Returns non-zero on success (parallels PyArg_ParseTuple). */ static int -parse_time_t_args(PyObject *args, char *format, time_t *pwhen) +parse_time_t_args(PyObject *args, const char *format, time_t *pwhen) { PyObject *ot = NULL; time_t whent; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 40c1760..c1a9be9 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -89,7 +89,7 @@ Xxo_getattro(XxoObject *self, PyObject *name) } static int -Xxo_setattr(XxoObject *self, char *name, PyObject *v) +Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 85230d9..0764407 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -76,7 +76,7 @@ Xxo_getattro(XxoObject *self, PyObject *name) } static int -Xxo_setattr(XxoObject *self, char *name, PyObject *v) +Xxo_setattr(XxoObject *self, const char *name, PyObject *v) { if (self->x_attr == NULL) { self->x_attr = PyDict_New(); diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 7220faf..87c8cfc 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -815,7 +815,7 @@ static PyTypeObject ZipImporter_Type = { 4 bytes, encoded as little endian. This partially reimplements marshal.c:r_long() */ static long -get_long(unsigned char *buf) { +get_long(const unsigned char *buf) { long x; x = buf[0]; x |= (long)buf[1] << 8; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a15fdb2..7d2f55a 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -53,7 +53,7 @@ typedef struct } compobject; static void -zlib_error(z_stream zst, int err, char *msg) +zlib_error(z_stream zst, int err, const char *msg) { const char *zmsg = Z_NULL; /* In case of a version mismatch, zst.msg won't be initialized. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 96ab57d..9e8ba39 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2576,8 +2576,8 @@ bytearray_remove_impl(PyByteArrayObject *self, int value) /* XXX These two helpers could be optimized if argsize == 1 */ static Py_ssize_t -lstrip_helper(char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) +lstrip_helper(const char *myptr, Py_ssize_t mysize, + const void *argptr, Py_ssize_t argsize) { Py_ssize_t i = 0; while (i < mysize && memchr(argptr, (unsigned char) myptr[i], argsize)) @@ -2586,8 +2586,8 @@ lstrip_helper(char *myptr, Py_ssize_t mysize, } static Py_ssize_t -rstrip_helper(char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) +rstrip_helper(const char *myptr, Py_ssize_t mysize, + const void *argptr, Py_ssize_t argsize) { Py_ssize_t i = mysize - 1; while (i >= 0 && memchr(argptr, (unsigned char) myptr[i], argsize)) diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index a299915..d025351 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -277,7 +277,7 @@ Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ characters, all remaining cased characters have lowercase."); void -_Py_bytes_title(char *result, char *s, Py_ssize_t len) +_Py_bytes_title(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; int previous_is_cased = 0; @@ -306,7 +306,7 @@ Return a copy of B with only its first character capitalized (ASCII)\n\ and the rest lower-cased."); void -_Py_bytes_capitalize(char *result, char *s, Py_ssize_t len) +_Py_bytes_capitalize(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; @@ -336,7 +336,7 @@ Return a copy of B with uppercase ASCII characters converted\n\ to lowercase ASCII and vice versa."); void -_Py_bytes_swapcase(char *result, char *s, Py_ssize_t len) +_Py_bytes_swapcase(char *result, const char *s, Py_ssize_t len) { Py_ssize_t i; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index f980516..602dea6 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -308,7 +308,7 @@ PyBytes_FromFormatV(const char *format, va_list vargs) { Py_ssize_t i; - p = va_arg(vargs, char*); + p = va_arg(vargs, const char*); i = strlen(p); if (prec > 0 && i > prec) i = prec; diff --git a/Objects/descrobject.c b/Objects/descrobject.c index da11f6b..2002d21 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -22,7 +22,7 @@ descr_name(PyDescrObject *descr) } static PyObject * -descr_repr(PyDescrObject *descr, char *format) +descr_repr(PyDescrObject *descr, const char *format) { PyObject *name = NULL; if (descr->d_name != NULL && PyUnicode_Check(descr->d_name)) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6f57db0..f1f31ed 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1925,7 +1925,7 @@ dict_fromkeys_impl(PyTypeObject *type, PyObject *iterable, PyObject *value) } static int -dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, char *methname) +dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, const char *methname) { PyObject *arg = NULL; int result = 0; diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 10162cb..e355a83 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1133,7 +1133,7 @@ get_native_fmtchar(char *result, const char *fmt) return -1; } -Py_LOCAL_INLINE(char *) +Py_LOCAL_INLINE(const char *) get_native_fmtstr(const char *fmt) { int at = 0; @@ -1221,7 +1221,7 @@ cast_to_1D(PyMemoryViewObject *mv, PyObject *format) goto out; } - view->format = get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); + view->format = (char *)get_native_fmtstr(PyBytes_AS_STRING(asciifmt)); if (view->format == NULL) { /* NOT_REACHED: get_native_fmtchar() already validates the format. */ PyErr_SetString(PyExc_RuntimeError, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b4d23b2..2e75ec3 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -564,7 +564,7 @@ type_get_bases(PyTypeObject *type, void *context) static PyTypeObject *best_base(PyObject *); static int mro_internal(PyTypeObject *, PyObject **); Py_LOCAL_INLINE(int) type_is_subtype_base_chain(PyTypeObject *, PyTypeObject *); -static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, char *); +static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *); static int add_subclass(PyTypeObject*, PyTypeObject*); static int add_all_subclasses(PyTypeObject *type, PyObject *bases); static void remove_subclass(PyTypeObject *, PyTypeObject *); @@ -1435,7 +1435,7 @@ _PyObject_LookupSpecial(PyObject *self, _Py_Identifier *attrid) as lookup_method to cache the interned name string object. */ static PyObject * -call_method(PyObject *o, _Py_Identifier *nameid, char *format, ...) +call_method(PyObject *o, _Py_Identifier *nameid, const char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; @@ -1471,7 +1471,7 @@ call_method(PyObject *o, _Py_Identifier *nameid, char *format, ...) /* Clone of call_method() that returns NotImplemented when the lookup fails. */ static PyObject * -call_maybe(PyObject *o, _Py_Identifier *nameid, char *format, ...) +call_maybe(PyObject *o, _Py_Identifier *nameid, const char *format, ...) { va_list va; PyObject *args, *func = 0, *retval; @@ -3609,7 +3609,7 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) } static int -compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, char* attr) +compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* attr) { PyTypeObject *newbase, *oldbase; @@ -5348,7 +5348,7 @@ wrap_delitem(PyObject *self, PyObject *args, void *wrapped) /* Helper to check for object.__setattr__ or __delattr__ applied to a type. This is called the Carlo Verre hack after its discoverer. */ static int -hackcheck(PyObject *self, setattrofunc func, char *what) +hackcheck(PyObject *self, setattrofunc func, const char *what) { PyTypeObject *type = Py_TYPE(self); while (type && type->tp_flags & Py_TPFLAGS_HEAPTYPE) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index fef184a..bda2469 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -6986,7 +6986,7 @@ PyUnicode_AsASCIIString(PyObject *unicode) # define WC_ERR_INVALID_CHARS 0x0080 #endif -static char* +static const char* code_page_name(UINT code_page, PyObject **obj) { *obj = NULL; @@ -7094,7 +7094,7 @@ decode_code_page_errors(UINT code_page, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; DWORD err; int ret = -1; @@ -7438,7 +7438,7 @@ encode_code_page_errors(UINT code_page, PyObject **outbytes, PyObject *errorHandler = NULL; PyObject *exc = NULL; PyObject *encoding_obj = NULL; - char *encoding; + const char *encoding; Py_ssize_t newpos, newoutsize; PyObject *rep; int ret = -1; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index d4d52e6..f42fe3d 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -265,7 +265,7 @@ insert_head(PyWeakReference *newref, PyWeakReference **list) } static int -parse_weakref_init_args(char *funcname, PyObject *args, PyObject *kwargs, +parse_weakref_init_args(const char *funcname, PyObject *args, PyObject *kwargs, PyObject **obp, PyObject **callbackp) { /* XXX Should check that kwargs == NULL or is empty. */ diff --git a/Parser/pgen.c b/Parser/pgen.c index 6ecb311..be35e02 100644 --- a/Parser/pgen.c +++ b/Parser/pgen.c @@ -379,7 +379,7 @@ typedef struct _ss_dfa { /* Forward */ static void printssdfa(int xx_nstates, ss_state *xx_state, int nbits, - labellist *ll, char *msg); + labellist *ll, const char *msg); static void simplify(int xx_nstates, ss_state *xx_state); static void convert(dfa *d, int xx_nstates, ss_state *xx_state); @@ -494,7 +494,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d) static void printssdfa(int xx_nstates, ss_state *xx_state, int nbits, - labellist *ll, char *msg) + labellist *ll, const char *msg) { int i, ibit, iarc; ss_state *yy; diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 0f055d6..3ca4afe 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -27,7 +27,7 @@ int Py_VerboseFlag; int Py_IgnoreEnvironmentFlag; /* Forward */ -grammar *getgrammar(char *filename); +grammar *getgrammar(const char *filename); void Py_Exit(int) _Py_NO_RETURN; @@ -76,7 +76,7 @@ main(int argc, char **argv) } grammar * -getgrammar(char *filename) +getgrammar(const char *filename) { FILE *fp; node *n; diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 9ca3cb4..be7cf49 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -202,8 +202,8 @@ error_ret(struct tok_state *tok) /* XXX */ } -static char * -get_normal_name(char *s) /* for utf-8 and latin-1 */ +static const char * +get_normal_name(const char *s) /* for utf-8 and latin-1 */ { char buf[13]; int i; @@ -264,7 +264,7 @@ get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *t if (begin < t) { char* r = new_string(begin, t - begin, tok); - char* q; + const char* q; if (!r) return 0; q = get_normal_name(r); diff --git a/Python/_warnings.c b/Python/_warnings.c index 978bad1..daa1355 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -921,7 +921,7 @@ PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) #undef PyErr_Warn PyAPI_FUNC(int) -PyErr_Warn(PyObject *category, char *text) +PyErr_Warn(PyObject *category, const char *text) { return PyErr_WarnEx(category, text, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index dd9360c..6f74e86 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -125,7 +125,7 @@ static PyObject * load_args(PyObject ***, int); #ifdef LLTRACE static int lltrace; -static int prtrace(PyObject *, char *); +static int prtrace(PyObject *, const char *); #endif static int call_trace(Py_tracefunc, PyObject *, PyThreadState *, PyFrameObject *, @@ -4308,7 +4308,7 @@ Error: #ifdef LLTRACE static int -prtrace(PyObject *v, char *str) +prtrace(PyObject *v, const char *str) { printf("%s ", str); if (PyObject_Print(v, stdout, 0) != 0) diff --git a/Python/dtoa.c b/Python/dtoa.c index 3121cd6..e0665b6 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -2315,7 +2315,7 @@ rv_alloc(int i) } static char * -nrv_alloc(char *s, char **rve, int n) +nrv_alloc(const char *s, char **rve, int n) { char *rv, *t; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 9fb0525..05050cf 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -41,7 +41,7 @@ const char *_PyImport_DynLoadFiletab[] = { /* Case insensitive string compare, to avoid any dependencies on particular C RTL implementations */ -static int strcasecmp (char *string1, char *string2) +static int strcasecmp (const char *string1, const char *string2) { int first, second; diff --git a/Python/getargs.c b/Python/getargs.c index c365fc2..7d45785 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -20,12 +20,12 @@ int PyArg_VaParseTupleAndKeywords(PyObject *, PyObject *, #ifdef HAVE_DECLSPEC_DLL /* Export functions */ -PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, char *, ...); -PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, char *, ...); +PyAPI_FUNC(int) _PyArg_Parse_SizeT(PyObject *, const char *, ...); +PyAPI_FUNC(int) _PyArg_ParseTuple_SizeT(PyObject *, const char *, ...); PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, ...); PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); -PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, char *, va_list); +PyAPI_FUNC(int) _PyArg_VaParse_SizeT(PyObject *, const char *, va_list); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *, PyObject *, const char *, char **, va_list); #endif @@ -56,18 +56,18 @@ typedef struct { /* Forward */ static int vgetargs1(PyObject *, const char *, va_list *, int); static void seterror(Py_ssize_t, const char *, int *, const char *, const char *); -static char *convertitem(PyObject *, const char **, va_list *, int, int *, - char *, size_t, freelist_t *); -static char *converttuple(PyObject *, const char **, va_list *, int, - int *, char *, size_t, int, freelist_t *); -static char *convertsimple(PyObject *, const char **, va_list *, int, char *, - size_t, freelist_t *); -static Py_ssize_t convertbuffer(PyObject *, void **p, char **); -static int getbuffer(PyObject *, Py_buffer *, char**); +static const char *convertitem(PyObject *, const char **, va_list *, int, int *, + char *, size_t, freelist_t *); +static const char *converttuple(PyObject *, const char **, va_list *, int, + int *, char *, size_t, int, freelist_t *); +static const char *convertsimple(PyObject *, const char **, va_list *, int, + char *, size_t, freelist_t *); +static Py_ssize_t convertbuffer(PyObject *, void **p, const char **); +static int getbuffer(PyObject *, Py_buffer *, const char**); static int vgetargskeywords(PyObject *, PyObject *, const char *, char **, va_list *, int); -static char *skipitem(const char **, va_list *, int); +static const char *skipitem(const char **, va_list *, int); int PyArg_Parse(PyObject *args, const char *format, ...) @@ -82,7 +82,7 @@ PyArg_Parse(PyObject *args, const char *format, ...) } int -_PyArg_Parse_SizeT(PyObject *args, char *format, ...) +_PyArg_Parse_SizeT(PyObject *args, const char *format, ...) { int retval; va_list va; @@ -107,7 +107,7 @@ PyArg_ParseTuple(PyObject *args, const char *format, ...) } int -_PyArg_ParseTuple_SizeT(PyObject *args, char *format, ...) +_PyArg_ParseTuple_SizeT(PyObject *args, const char *format, ...) { int retval; va_list va; @@ -130,7 +130,7 @@ PyArg_VaParse(PyObject *args, const char *format, va_list va) } int -_PyArg_VaParse_SizeT(PyObject *args, char *format, va_list va) +_PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va) { va_list lva; @@ -208,7 +208,7 @@ vgetargs1(PyObject *args, const char *format, va_list *p_va, int flags) int endfmt = 0; const char *formatsave = format; Py_ssize_t i, len; - char *msg; + const char *msg; int compat = flags & FLAG_COMPAT; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; @@ -416,7 +416,7 @@ seterror(Py_ssize_t iarg, const char *msg, int *levels, const char *fname, and msgbuf is returned. */ -static char * +static const char * converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, int *levels, char *msgbuf, size_t bufsize, int toplevel, freelist_t *freelist) @@ -474,7 +474,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, format = *p_format; for (i = 0; i < n; i++) { - char *msg; + const char *msg; PyObject *item; item = PySequence_GetItem(arg, i); if (item == NULL) { @@ -501,11 +501,11 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* Convert a single item. */ -static char * +static const char * convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, int *levels, char *msgbuf, size_t bufsize, freelist_t *freelist) { - char *msg; + const char *msg; const char *format = *p_format; if (*format == '(' /* ')' */) { @@ -530,7 +530,7 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* Format an error message generated by convertsimple(). */ -static char * +static const char * converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); @@ -566,7 +566,7 @@ float_argument_error(PyObject *arg) When you add new format codes, please don't forget poor skipitem() below. */ -static char * +static const char * convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { @@ -851,7 +851,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'y': {/* any bytes-like object */ void **p = (void **)va_arg(*p_va, char **); - char *buf; + const char *buf; Py_ssize_t count; if (*format == '*') { if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) @@ -898,7 +898,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, PyBuffer_FillInfo(p, arg, sarg, len, 1, 0); } else { /* any bytes-like object */ - char *buf; + const char *buf; if (getbuffer(arg, p, &buf) < 0) return converterr(buf, arg, msgbuf, bufsize); } @@ -928,7 +928,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { /* read-only bytes-like object */ /* XXX Really? */ - char *buf; + const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) return converterr(buf, arg, msgbuf, bufsize); @@ -1275,7 +1275,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } static Py_ssize_t -convertbuffer(PyObject *arg, void **p, char **errmsg) +convertbuffer(PyObject *arg, void **p, const char **errmsg) { PyBufferProcs *pb = Py_TYPE(arg)->tp_as_buffer; Py_ssize_t count; @@ -1297,7 +1297,7 @@ convertbuffer(PyObject *arg, void **p, char **errmsg) } static int -getbuffer(PyObject *arg, Py_buffer *view, char **errmsg) +getbuffer(PyObject *arg, Py_buffer *view, const char **errmsg) { if (PyObject_GetBuffer(arg, view, PyBUF_SIMPLE) != 0) { *errmsg = "bytes-like object"; @@ -1629,7 +1629,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } -static char * +static const char * skipitem(const char **p_format, va_list *p_va, int flags) { const char *format = *p_format; @@ -1722,7 +1722,7 @@ skipitem(const char **p_format, va_list *p_va, int flags) case '(': /* bypass tuple, not handled at all previously */ { - char *msg; + const char *msg; for (;;) { if (*format==')') break; diff --git a/Python/marshal.c b/Python/marshal.c index 5b8de99..589eb80 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -643,7 +643,7 @@ typedef struct { PyObject *refs; /* a list */ } RFILE; -static char * +static const char * r_string(Py_ssize_t n, RFILE *p) { Py_ssize_t read = -1; @@ -729,7 +729,7 @@ r_byte(RFILE *p) c = getc(p->fp); } else { - char *ptr = r_string(1, p); + const char *ptr = r_string(1, p); if (ptr != NULL) c = *(unsigned char *) ptr; } @@ -740,9 +740,9 @@ static int r_short(RFILE *p) { short x = -1; - unsigned char *buffer; + const unsigned char *buffer; - buffer = (unsigned char *) r_string(2, p); + buffer = (const unsigned char *) r_string(2, p); if (buffer != NULL) { x = buffer[0]; x |= buffer[1] << 8; @@ -756,9 +756,9 @@ static long r_long(RFILE *p) { long x = -1; - unsigned char *buffer; + const unsigned char *buffer; - buffer = (unsigned char *) r_string(4, p); + buffer = (const unsigned char *) r_string(4, p); if (buffer != NULL) { x = buffer[0]; x |= (long)buffer[1] << 8; @@ -978,7 +978,8 @@ r_object(RFILE *p) case TYPE_FLOAT: { - char buf[256], *ptr; + char buf[256]; + const char *ptr; double dx; n = r_byte(p); if (n == EOF) { @@ -1001,9 +1002,9 @@ r_object(RFILE *p) case TYPE_BINARY_FLOAT: { - unsigned char *buf; + const unsigned char *buf; double x; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; x = _PyFloat_Unpack8(buf, 1); @@ -1016,7 +1017,8 @@ r_object(RFILE *p) case TYPE_COMPLEX: { - char buf[256], *ptr; + char buf[256]; + const char *ptr; Py_complex c; n = r_byte(p); if (n == EOF) { @@ -1053,15 +1055,15 @@ r_object(RFILE *p) case TYPE_BINARY_COMPLEX: { - unsigned char *buf; + const unsigned char *buf; Py_complex c; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; c.real = _PyFloat_Unpack8(buf, 1); if (c.real == -1.0 && PyErr_Occurred()) break; - buf = (unsigned char *) r_string(8, p); + buf = (const unsigned char *) r_string(8, p); if (buf == NULL) break; c.imag = _PyFloat_Unpack8(buf, 1); @@ -1074,7 +1076,7 @@ r_object(RFILE *p) case TYPE_STRING: { - char *ptr; + const char *ptr; n = r_long(p); if (PyErr_Occurred()) break; @@ -1119,7 +1121,7 @@ r_object(RFILE *p) } _read_ascii: { - char *ptr; + const char *ptr; ptr = r_string(n, p); if (ptr == NULL) break; @@ -1137,7 +1139,7 @@ r_object(RFILE *p) is_interned = 1; case TYPE_UNICODE: { - char *buffer; + const char *buffer; n = r_long(p); if (PyErr_Occurred()) diff --git a/Python/modsupport.c b/Python/modsupport.c index 6c938dd..7e6a65b 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -301,7 +301,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'U': /* XXX deprecated alias */ { PyObject *v; - char *str = va_arg(*p_va, char *); + const char *str = va_arg(*p_va, const char *); Py_ssize_t n; if (**p_format == '#') { ++*p_format; @@ -334,7 +334,7 @@ do_mkvalue(const char **p_format, va_list *p_va, int flags) case 'y': { PyObject *v; - char *str = va_arg(*p_va, char *); + const char *str = va_arg(*p_va, const char *); Py_ssize_t n; if (**p_format == '#') { ++*p_format; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b7f6ec8..c84c46a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1004,8 +1004,8 @@ is_valid_fd(int fd) /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(PyObject* io, - int fd, int write_mode, char* name, - char* encoding, char* errors) + int fd, int write_mode, const char* name, + const char* encoding, const char* errors) { PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res; const char* mode; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index cfe197b..8829699 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1138,8 +1138,8 @@ PyParser_ASTFromString(const char *s, const char *filename_str, int start, mod_ty PyParser_ASTFromFileObject(FILE *fp, PyObject *filename, const char* enc, - int start, char *ps1, - char *ps2, PyCompilerFlags *flags, int *errcode, + int start, const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { mod_ty mod; @@ -1171,8 +1171,8 @@ PyParser_ASTFromFileObject(FILE *fp, PyObject *filename, const char* enc, mod_ty PyParser_ASTFromFile(FILE *fp, const char *filename_str, const char* enc, - int start, char *ps1, - char *ps2, PyCompilerFlags *flags, int *errcode, + int start, const char *ps1, + const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { mod_ty mod; diff --git a/Python/symtable.c b/Python/symtable.c index 8431d51..806364a 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -160,7 +160,7 @@ PyTypeObject PySTEntry_Type = { }; static int symtable_analyze(struct symtable *st); -static int symtable_warn(struct symtable *st, char *msg, int lineno); +static int symtable_warn(struct symtable *st, const char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, int lineno, int col_offset); @@ -903,7 +903,7 @@ symtable_analyze(struct symtable *st) static int -symtable_warn(struct symtable *st, char *msg, int lineno) +symtable_warn(struct symtable *st, const char *msg, int lineno) { PyObject *message = PyUnicode_FromString(msg); if (message == NULL) -- cgit v0.12 From aab9f46c6d5508383709f8722a9328fa76dd8c1b Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Sat, 26 Dec 2015 12:35:47 +0000 Subject: Closes #25789: Improved buffering behaviour in launcher. --- PC/launcher.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PC/launcher.c b/PC/launcher.c index f7a0917..02cd9b9 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -98,7 +98,7 @@ error(int rc, wchar_t * format, ... ) MessageBox(NULL, message, TEXT("Python Launcher is sorry to say ..."), MB_OK); #endif - ExitProcess(rc); + exit(rc); } /* @@ -652,7 +652,7 @@ run_child(wchar_t * cmdline) if (!ok) error(RC_CREATE_PROCESS, L"Failed to get exit code of process"); debug(L"child process exit code: %d\n", rc); - ExitProcess(rc); + exit(rc); } static void @@ -1357,6 +1357,7 @@ process(int argc, wchar_t ** argv) wchar_t * av[2]; #endif + setvbuf(stderr, (char *)NULL, _IONBF, 0); wp = get_env(L"PYLAUNCH_DEBUG"); if ((wp != NULL) && (*wp != L'\0')) log_fp = stderr; -- cgit v0.12 From 1ed017ae92b32b27186d5793f6e58c526f350a2b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 27 Dec 2015 15:51:32 +0200 Subject: Issue #20440: Cleaning up the code by using Py_SETREF and Py_CLEAR. Old code is correct, but with Py_SETREF and Py_CLEAR it can be cleaner. This patch doesn't fix bugs and hence there is no need to backport it. --- Modules/_collectionsmodule.c | 5 +---- Modules/_ctypes/_ctypes.c | 10 ++++------ Modules/_elementtree.c | 6 +----- Modules/_io/bytesio.c | 6 ++---- Modules/itertoolsmodule.c | 22 ++++++---------------- Modules/pyexpat.c | 17 ++++------------- Objects/listobject.c | 10 ++-------- Objects/tupleobject.c | 5 +---- Objects/typeobject.c | 13 ++----------- 9 files changed, 23 insertions(+), 71 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index e3d1910..6a53584 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1222,7 +1222,6 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) static int deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) { - PyObject *old_value; block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1249,9 +1248,7 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) b = b->leftlink; } Py_INCREF(v); - old_value = b->data[i]; - b->data[i] = v; - Py_DECREF(old_value); + Py_SETREF(b->data[i], v); return 0; } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 5db9f90..dddeba0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5123,23 +5123,22 @@ int comerror_init(PyObject *self, PyObject *args, PyObject *kwds) { PyObject *hresult, *text, *details; - PyBaseExceptionObject *bself; PyObject *a; int status; if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) - return -1; + return -1; if (!PyArg_ParseTuple(args, "OOO:COMError", &hresult, &text, &details)) return -1; a = PySequence_GetSlice(args, 1, PySequence_Size(args)); if (!a) - return -1; + return -1; status = PyObject_SetAttrString(self, "args", a); Py_DECREF(a); if (status < 0) - return -1; + return -1; if (PyObject_SetAttrString(self, "hresult", hresult) < 0) return -1; @@ -5150,9 +5149,8 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) if (PyObject_SetAttrString(self, "details", details) < 0) return -1; - bself = (PyBaseExceptionObject *)self; Py_INCREF(args); - Py_SETREF(bself->args, args); + Py_SETREF((PyBaseExceptionObject *)self->args, args); return 0; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index f16d48f..580c53a 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2338,13 +2338,9 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self, PyObject *element_factory) /*[clinic end generated code: output=91cfa7558970ee96 input=1b424eeefc35249c]*/ { - PyObject *tmp; - if (element_factory) { Py_INCREF(element_factory); - tmp = self->element_factory; - self->element_factory = element_factory; - Py_XDECREF(tmp); + Py_SETREF(self->element_factory, element_factory); } return 0; diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 99e71bc..6333e46 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -87,7 +87,7 @@ scan_eol(bytesio *self, Py_ssize_t len) static int unshare_buffer(bytesio *self, size_t size) { - PyObject *new_buf, *old_buf; + PyObject *new_buf; assert(SHARED_BUF(self)); assert(self->exports == 0); assert(size >= (size_t)self->string_size); @@ -96,9 +96,7 @@ unshare_buffer(bytesio *self, size_t size) return -1; memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), self->string_size); - old_buf = self->buf; - self->buf = new_buf; - Py_DECREF(old_buf); + Py_SETREF(self->buf, new_buf); return 0; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 2bd0594..a4ffa06 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -77,7 +77,7 @@ groupby_traverse(groupbyobject *gbo, visitproc visit, void *arg) static PyObject * groupby_next(groupbyobject *gbo) { - PyObject *newvalue, *newkey, *r, *grouper, *tmp; + PyObject *newvalue, *newkey, *r, *grouper; /* skip to next iteration group */ for (;;) { @@ -110,19 +110,12 @@ groupby_next(groupbyobject *gbo) } } - tmp = gbo->currkey; - gbo->currkey = newkey; - Py_XDECREF(tmp); - - tmp = gbo->currvalue; - gbo->currvalue = newvalue; - Py_XDECREF(tmp); + Py_SETREF(gbo->currkey, newkey); + Py_SETREF(gbo->currvalue, newvalue); } Py_INCREF(gbo->currkey); - tmp = gbo->tgtkey; - gbo->tgtkey = gbo->currkey; - Py_XDECREF(tmp); + Py_SETREF(gbo->tgtkey, gbo->currkey); grouper = _grouper_create(gbo, gbo->tgtkey); if (grouper == NULL) @@ -3445,7 +3438,7 @@ accumulate_traverse(accumulateobject *lz, visitproc visit, void *arg) static PyObject * accumulate_next(accumulateobject *lz) { - PyObject *val, *oldtotal, *newtotal; + PyObject *val, *newtotal; val = (*Py_TYPE(lz->it)->tp_iternext)(lz->it); if (val == NULL) @@ -3465,11 +3458,8 @@ accumulate_next(accumulateobject *lz) if (newtotal == NULL) return NULL; - oldtotal = lz->total; - lz->total = newtotal; - Py_DECREF(oldtotal); - Py_INCREF(newtotal); + Py_SETREF(lz->total, newtotal); return newtotal; } diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 9267c69..c6d3515 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1226,12 +1226,8 @@ xmlparse_dealloc(xmlparseobject *self) self->itself = NULL; if (self->handlers != NULL) { - PyObject *temp; - for (i = 0; handler_info[i].name != NULL; i++) { - temp = self->handlers[i]; - self->handlers[i] = NULL; - Py_XDECREF(temp); - } + for (i = 0; handler_info[i].name != NULL; i++) + Py_CLEAR(self->handlers[i]); PyMem_Free(self->handlers); self->handlers = NULL; } @@ -1345,7 +1341,6 @@ sethandler(xmlparseobject *self, PyObject *name, PyObject* v) int handlernum = handlername2int(name); if (handlernum >= 0) { xmlhandler c_handler = NULL; - PyObject *temp = self->handlers[handlernum]; if (v == Py_None) { /* If this is the character data handler, and a character @@ -1367,8 +1362,7 @@ sethandler(xmlparseobject *self, PyObject *name, PyObject* v) Py_INCREF(v); c_handler = handler_info[handlernum].handler; } - self->handlers[handlernum] = v; - Py_XDECREF(temp); + Py_SETREF(self->handlers[handlernum], v); handler_info[handlernum].setter(self->itself, c_handler); return 1; } @@ -1898,15 +1892,12 @@ static void clear_handlers(xmlparseobject *self, int initial) { int i = 0; - PyObject *temp; for (; handler_info[i].name != NULL; i++) { if (initial) self->handlers[i] = NULL; else { - temp = self->handlers[i]; - self->handlers[i] = NULL; - Py_XDECREF(temp); + Py_CLEAR(self->handlers[i]); handler_info[i].setter(self->itself, NULL); } } diff --git a/Objects/listobject.c b/Objects/listobject.c index eee7c68..fcc21cb 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -216,7 +216,6 @@ int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { - PyObject *olditem; PyObject **p; if (!PyList_Check(op)) { Py_XDECREF(newitem); @@ -230,9 +229,7 @@ PyList_SetItem(PyObject *op, Py_ssize_t i, return -1; } p = ((PyListObject *)op) -> ob_item + i; - olditem = *p; - *p = newitem; - Py_XDECREF(olditem); + Py_SETREF(*p, newitem); return 0; } @@ -730,7 +727,6 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) static int list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) { - PyObject *old_value; if (i < 0 || i >= Py_SIZE(a)) { PyErr_SetString(PyExc_IndexError, "list assignment index out of range"); @@ -739,9 +735,7 @@ list_ass_item(PyListObject *a, Py_ssize_t i, PyObject *v) if (v == NULL) return list_ass_slice(a, i, i+1, v); Py_INCREF(v); - old_value = a->ob_item[i]; - a->ob_item[i] = v; - Py_DECREF(old_value); + Py_SETREF(a->ob_item[i], v); return 0; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 7efa1a6..8e1b00b 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -149,7 +149,6 @@ PyTuple_GetItem(PyObject *op, Py_ssize_t i) int PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { - PyObject *olditem; PyObject **p; if (!PyTuple_Check(op) || op->ob_refcnt != 1) { Py_XDECREF(newitem); @@ -163,9 +162,7 @@ PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) return -1; } p = ((PyTupleObject *)op) -> ob_item + i; - olditem = *p; - *p = newitem; - Py_XDECREF(olditem); + Py_SETREF(*p, newitem); return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4fac9e8..78ef6ac 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -401,7 +401,6 @@ type_qualname(PyTypeObject *type, void *context) static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { - PyHeapTypeObject* et; char *tp_name; PyObject *tmp; @@ -430,17 +429,9 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) if (tp_name == NULL) return -1; - et = (PyHeapTypeObject*)type; - - Py_INCREF(value); - - /* Wait until et is a sane state before Py_DECREF'ing the old et->ht_name - value. (Bug #16447.) */ - tmp = et->ht_name; - et->ht_name = value; - type->tp_name = tp_name; - Py_DECREF(tmp); + Py_INCREF(value); + Py_SETREF(((PyHeapTypeObject*)type)->ht_name, value); return 0; } -- cgit v0.12 From 1e3c3e906c9f1fcc463cfb3641d078eec773666f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 27 Dec 2015 13:17:04 -0800 Subject: Issue #25768: Make compileall functions return booleans and document the return values as well as test them. Thanks to Nicholas Chammas for the bug report and initial patch. --- Doc/library/compileall.rst | 12 ++++++++---- Doc/whatsnew/3.6.rst | 5 +++++ Lib/compileall.py | 16 ++++++++-------- Lib/test/test_compileall.py | 26 ++++++++++++++++++++++++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c5736f2..679c2b4 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -103,7 +103,8 @@ Public functions .. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` - files along the way. + files along the way. Return a true value if all the files compiled successfully, + and a false value otherwise. The *maxlevels* parameter is used to limit the depth of the recursion; it defaults to ``10``. @@ -155,7 +156,8 @@ Public functions .. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1) - Compile the file with path *fullname*. + Compile the file with path *fullname*. Return a true value if the file + compiled successfully, and a false value otherwise. If *ddir* is given, it is prepended to the path to the file being compiled for use in compilation time tracebacks, and is also compiled in to the @@ -191,8 +193,10 @@ Public functions .. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1) - Byte-compile all the :file:`.py` files found along ``sys.path``. If - *skip_curdir* is true (the default), the current directory is not included + Byte-compile all the :file:`.py` files found along ``sys.path``. Return a + true value if all the files compiled successfully, and a false value otherwise. + + If *skip_curdir* is true (the default), the current directory is not included in the search. All other parameters are passed to the :func:`compile_dir` function. Note that unlike the other compile functions, ``maxlevels`` defaults to ``0``. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f97c70f..c660933 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -230,6 +230,11 @@ that may require changes to your code. Changes in the Python API ------------------------- +* The functions in the :mod:`compileall` module now return booleans instead + of ``1`` or ``0`` to represent success or failure, respectively. Thanks to + booleans being a subclass of integers, this should only be an issue if you + were doing identity checks for ``1`` or ``0``. See :issue:`25768`. + * Reading the :attr:`~urllib.parse.SplitResult.port` attribute of :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results now raises :exc:`ValueError` for out-of-range values, rather than diff --git a/Lib/compileall.py b/Lib/compileall.py index 0cc0c1d..67c5f5a 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -68,7 +68,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, """ files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, ddir=ddir) - success = 1 + success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') @@ -81,12 +81,12 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, legacy=legacy, optimize=optimize), files) - success = min(results, default=1) + success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, legacy, optimize): - success = 0 + success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, @@ -104,7 +104,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter """ - success = 1 + success = True name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) @@ -144,7 +144,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize) except py_compile.PyCompileError as err: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -157,7 +157,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: - success = 0 + success = False if quiet >= 2: return success elif quiet: @@ -167,7 +167,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, print(e.__class__.__name__ + ':', e) else: if ok == 0: - success = 0 + success = False return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, @@ -183,7 +183,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) """ - success = 1 + success = True for dir in sys.path: if (not dir or dir == os.curdir) and skip_curdir: if quiet < 2: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 2ce8a61..439c125 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -1,6 +1,7 @@ import sys import compileall import importlib.util +import test.test_importlib.util import os import pathlib import py_compile @@ -40,6 +41,11 @@ class CompileallTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.directory) + def add_bad_source_file(self): + self.bad_source_path = os.path.join(self.directory, '_test_bad.py') + with open(self.bad_source_path, 'w') as file: + file.write('x (\n') + def data(self): with open(self.bc_path, 'rb') as file: data = file.read(8) @@ -78,15 +84,31 @@ class CompileallTests(unittest.TestCase): os.unlink(fn) except: pass - compileall.compile_file(self.source_path, force=False, quiet=True) + self.assertTrue(compileall.compile_file(self.source_path, + force=False, quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) - compileall.compile_dir(self.directory, force=False, quiet=True) + self.assertTrue(compileall.compile_dir(self.directory, force=False, + quiet=True)) self.assertTrue(os.path.isfile(self.bc_path) and os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + # Test against bad files + self.add_bad_source_file() + self.assertFalse(compileall.compile_file(self.bad_source_path, + force=False, quiet=2)) + self.assertFalse(compileall.compile_dir(self.directory, + force=False, quiet=2)) + + def test_compile_path(self): + self.assertTrue(compileall.compile_path(quiet=2)) + + with test.test_importlib.util.import_state(path=[self.directory]): + self.add_bad_source_file() + self.assertFalse(compileall.compile_path(skip_curdir=False, + force=True, quiet=2)) def test_no_pycache_in_non_package(self): # Bug 8563 reported that __pycache__ directories got created by diff --git a/Misc/ACKS b/Misc/ACKS index 09ad15c..c1bf5d1 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -235,6 +235,7 @@ Octavian Cerna Michael Cetrulo Dave Chambers Pascal Chambon +Nicholas Chammas John Chandler Hye-Shik Chang Jeffrey Chang diff --git a/Misc/NEWS b/Misc/NEWS index f0b83b1..e795234 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,9 @@ Core and Builtins Library ------- +- Issue #25768: Have the functions in compileall return booleans instead of + ints and add proper documentation and tests for the return values. + - Issue #24103: Fixed possible use after free in ElementTree.XMLPullParser. - Issue #25860: os.fwalk() no longer skips remaining directories when error -- cgit v0.12 From ac1e7f6983ef4f0feef6389d458544bf924f6965 Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 28 Dec 2015 23:50:19 +0200 Subject: Remove duplicate method in test_pathlib. Initial patch by Navneet Suman. --- Lib/test/test_pathlib.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 1c53ab7..80cb97e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1437,14 +1437,14 @@ class _BasePathTest(object): self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) self.assertEqual(set(p.glob("../xyzzy")), set()) - def _check_resolve_relative(self, p, expected): - q = p.resolve() - self.assertEqual(q, expected) - def _check_resolve_absolute(self, p, expected): + def _check_resolve(self, p, expected): q = p.resolve() self.assertEqual(q, expected) + # this can be used to check both relative and absolute resolutions + _check_resolve_relative = _check_resolve_absolute = _check_resolve + @with_symlinks def test_resolve_common(self): P = self.cls -- cgit v0.12 From 53f2e0ad45e41c007f714e077ccf11643651ef28 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 28 Dec 2015 23:02:02 +0100 Subject: Issue #25928: Add Decimal.as_integer_ratio(). Python parts and docs by Mark Dickinson. --- Doc/library/decimal.rst | 13 +++++ Lib/_pydecimal.py | 52 +++++++++++++++++++ Lib/test/test_decimal.py | 33 ++++++++++++ Misc/NEWS | 2 + Modules/_decimal/_decimal.c | 101 +++++++++++++++++++++++++++++++++++++ Modules/_decimal/docstrings.h | 9 ++++ Modules/_decimal/tests/deccheck.py | 6 +-- 7 files changed, 213 insertions(+), 3 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 2de0ea0..14356aa 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -448,6 +448,19 @@ Decimal objects ``Decimal('321e+5').adjusted()`` returns seven. Used for determining the position of the most significant digit with respect to the decimal point. + .. method:: as_integer_ratio() + + Return a pair ``(n, d)`` of integers that represent the given + :class:`Decimal` instance as a fraction, in lowest terms and + with a positive denominator:: + + >>> Decimal('-3.14').as_integer_ratio() + (-157, 50) + + The conversion is exact. Raise OverflowError on infinities and ValueError + on NaNs. + + .. versionadded:: 3.6 .. method:: as_tuple() diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 05ba4ee..eb7bba8 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1010,6 +1010,58 @@ class Decimal(object): """ return DecimalTuple(self._sign, tuple(map(int, self._int)), self._exp) + def as_integer_ratio(self): + """Express a finite Decimal instance in the form n / d. + + Returns a pair (n, d) of integers. When called on an infinity + or NaN, raises OverflowError or ValueError respectively. + + >>> Decimal('3.14').as_integer_ratio() + (157, 50) + >>> Decimal('-123e5').as_integer_ratio() + (-12300000, 1) + >>> Decimal('0.00').as_integer_ratio() + (0, 1) + + """ + if self._is_special: + if self.is_nan(): + raise ValueError("Cannot pass NaN " + "to decimal.as_integer_ratio.") + else: + raise OverflowError("Cannot pass infinity " + "to decimal.as_integer_ratio.") + + if not self: + return 0, 1 + + # Find n, d in lowest terms such that abs(self) == n / d; + # we'll deal with the sign later. + n = int(self._int) + if self._exp >= 0: + # self is an integer. + n, d = n * 10**self._exp, 1 + else: + # Find d2, d5 such that abs(self) = n / (2**d2 * 5**d5). + d5 = -self._exp + while d5 > 0 and n % 5 == 0: + n //= 5 + d5 -= 1 + + # (n & -n).bit_length() - 1 counts trailing zeros in binary + # representation of n (provided n is nonzero). + d2 = -self._exp + shift2 = min((n & -n).bit_length() - 1, d2) + if shift2: + n >>= shift2 + d2 -= shift2 + + d = 5**d5 << d2 + + if self._sign: + n = -n + return n, d + def __repr__(self): """Represents the number as an instance of Decimal.""" # Invariant: eval(repr(d)) == d diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 137aaa5..f6d58ff 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -2047,6 +2047,39 @@ class UsabilityTest(unittest.TestCase): d = Decimal( (1, (0, 2, 7, 1), 'F') ) self.assertEqual(d.as_tuple(), (1, (0,), 'F')) + def test_as_integer_ratio(self): + Decimal = self.decimal.Decimal + + # exceptional cases + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('inf')) + self.assertRaises(OverflowError, + Decimal.as_integer_ratio, Decimal('-inf')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('-nan')) + self.assertRaises(ValueError, + Decimal.as_integer_ratio, Decimal('snan123')) + + for exp in range(-4, 2): + for coeff in range(1000): + for sign in '+', '-': + d = Decimal('%s%dE%d' % (sign, coeff, exp)) + pq = d.as_integer_ratio() + p, q = pq + + # check return type + self.assertIsInstance(pq, tuple) + self.assertIsInstance(p, int) + self.assertIsInstance(q, int) + + # check normalization: q should be positive; + # p should be relatively prime to q. + self.assertGreater(q, 0) + self.assertEqual(math.gcd(p, q), 1) + + # check that p/q actually gives the correct value + self.assertEqual(Decimal(p) / Decimal(q), d) + def test_subclassing(self): # Different behaviours when subclassing Decimal Decimal = self.decimal.Decimal diff --git a/Misc/NEWS b/Misc/NEWS index e795234..821904f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,8 @@ Core and Builtins Library ------- +- Issue #25928: Add Decimal.as_integer_ratio(). + - Issue #25768: Have the functions in compileall return booleans instead of ints and add proper documentation and tests for the return values. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 112b44f..3c2ad85 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -3380,6 +3380,106 @@ dec_as_long(PyObject *dec, PyObject *context, int round) return (PyObject *) pylong; } +/* Convert a Decimal to its exact integer ratio representation. */ +static PyObject * +dec_as_integer_ratio(PyObject *self, PyObject *args UNUSED) +{ + PyObject *numerator = NULL; + PyObject *denominator = NULL; + PyObject *exponent = NULL; + PyObject *result = NULL; + PyObject *tmp; + mpd_ssize_t exp; + PyObject *context; + uint32_t status = 0; + PyNumberMethods *long_methods = PyLong_Type.tp_as_number; + + if (mpd_isspecial(MPD(self))) { + if (mpd_isnan(MPD(self))) { + PyErr_SetString(PyExc_ValueError, + "cannot convert NaN to integer ratio"); + } + else { + PyErr_SetString(PyExc_OverflowError, + "cannot convert Infinity to integer ratio"); + } + return NULL; + } + + CURRENT_CONTEXT(context); + + tmp = dec_alloc(); + if (tmp == NULL) { + return NULL; + } + + if (!mpd_qcopy(MPD(tmp), MPD(self), &status)) { + Py_DECREF(tmp); + PyErr_NoMemory(); + return NULL; + } + + exp = mpd_iszero(MPD(tmp)) ? 0 : MPD(tmp)->exp; + MPD(tmp)->exp = 0; + + /* context and rounding are unused here: the conversion is exact */ + numerator = dec_as_long(tmp, context, MPD_ROUND_FLOOR); + Py_DECREF(tmp); + if (numerator == NULL) { + goto error; + } + + exponent = PyLong_FromSsize_t(exp < 0 ? -exp : exp); + if (exponent == NULL) { + goto error; + } + + tmp = PyLong_FromLong(10); + if (tmp == NULL) { + goto error; + } + + Py_SETREF(exponent, long_methods->nb_power(tmp, exponent, Py_None)); + Py_DECREF(tmp); + if (exponent == NULL) { + goto error; + } + + if (exp >= 0) { + Py_SETREF(numerator, long_methods->nb_multiply(numerator, exponent)); + if (numerator == NULL) { + goto error; + } + denominator = PyLong_FromLong(1); + if (denominator == NULL) { + goto error; + } + } + else { + denominator = exponent; + exponent = NULL; + tmp = _PyLong_GCD(numerator, denominator); + if (tmp == NULL) { + goto error; + } + Py_SETREF(numerator, long_methods->nb_floor_divide(numerator, tmp)); + Py_SETREF(denominator, long_methods->nb_floor_divide(denominator, tmp)); + Py_DECREF(tmp); + if (numerator == NULL || denominator == NULL) { + goto error; + } + } + + result = PyTuple_Pack(2, numerator, denominator); + + +error: + Py_XDECREF(exponent); + Py_XDECREF(denominator); + Py_XDECREF(numerator); + return result; +} + static PyObject * PyDec_ToIntegralValue(PyObject *dec, PyObject *args, PyObject *kwds) { @@ -4688,6 +4788,7 @@ static PyMethodDef dec_methods [] = /* Miscellaneous */ { "from_float", dec_from_float, METH_O|METH_CLASS, doc_from_float }, { "as_tuple", PyDec_AsTuple, METH_NOARGS, doc_as_tuple }, + { "as_integer_ratio", dec_as_integer_ratio, METH_NOARGS, doc_as_integer_ratio }, /* Special methods */ { "__copy__", dec_copy, METH_NOARGS, NULL }, diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index 71029a9..f7fd6e7 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -70,6 +70,15 @@ PyDoc_STRVAR(doc_as_tuple, Return a tuple representation of the number.\n\ \n"); +PyDoc_STRVAR(doc_as_integer_ratio, +"as_integer_ratio($self, /)\n--\n\n\ +Decimal.as_integer_ratio() -> (int, int)\n\ +\n\ +Return a pair of integers, whose ratio is exactly equal to the original\n\ +Decimal and with a positive denominator. The ratio is in lowest terms.\n\ +Raise OverflowError on infinities and a ValueError on NaNs.\n\ +\n"); + PyDoc_STRVAR(doc_canonical, "canonical($self, /)\n--\n\n\ Return the canonical encoding of the argument. Currently, the encoding\n\ diff --git a/Modules/_decimal/tests/deccheck.py b/Modules/_decimal/tests/deccheck.py index ab7d5bd..f907531 100644 --- a/Modules/_decimal/tests/deccheck.py +++ b/Modules/_decimal/tests/deccheck.py @@ -50,8 +50,8 @@ Functions = { '__abs__', '__bool__', '__ceil__', '__complex__', '__copy__', '__floor__', '__float__', '__hash__', '__int__', '__neg__', '__pos__', '__reduce__', '__repr__', '__str__', '__trunc__', - 'adjusted', 'as_tuple', 'canonical', 'conjugate', 'copy_abs', - 'copy_negate', 'is_canonical', 'is_finite', 'is_infinite', + 'adjusted', 'as_integer_ratio', 'as_tuple', 'canonical', 'conjugate', + 'copy_abs', 'copy_negate', 'is_canonical', 'is_finite', 'is_infinite', 'is_nan', 'is_qnan', 'is_signed', 'is_snan', 'is_zero', 'radix' ), # Unary with optional context: @@ -128,7 +128,7 @@ ContextFunctions = { # Functions that require a restricted exponent range for reasonable runtimes. UnaryRestricted = [ '__ceil__', '__floor__', '__int__', '__trunc__', - 'to_integral', 'to_integral_value' + 'as_integer_ratio', 'to_integral', 'to_integral_value' ] BinaryRestricted = ['__round__'] -- cgit v0.12 From 6f5470880eb13cb28d189953375a79b61756ba78 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Mon, 28 Dec 2015 23:17:05 +0100 Subject: Fix typo. --- Misc/NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS b/Misc/NEWS index 7ee0355..b042090 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -126,7 +126,7 @@ Library - Issue #25928: Add Decimal.as_integer_ratio(). - Issue #25447: Copying the lru_cache() wrapper object now always works, - independedly from the type of the wrapped object (by returning the original + independently from the type of the wrapped object (by returning the original object unchanged). - Issue #25768: Have the functions in compileall return booleans instead of -- cgit v0.12 From 2335f44d8e3b71814b26583f4942ee16738b7486 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 00:11:36 +0100 Subject: Issue #25928: Temporarily disable some tests in test_statistics in order to sort out its assumptions about the as_integer_ratio() interface. --- Lib/test/test_statistics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 0089ae8..9a54fe1 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -699,6 +699,7 @@ class ExactRatioTest(unittest.TestCase): num, den = statistics._exact_ratio(x) self.assertEqual(x, num/den) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal(self): D = Decimal _exact_ratio = statistics._exact_ratio @@ -730,6 +731,7 @@ class ExactRatioTest(unittest.TestCase): self.assertIs(ratio[1], None) self.assertEqual(type(ratio[0]), type(nan)) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_nan(self): NAN = Decimal("NAN") sNAN = Decimal("sNAN") @@ -1258,6 +1260,7 @@ class SumSpecialValues(NumericTestCase): with decimal.localcontext(decimal.BasicContext): self.assertRaises(decimal.InvalidOperation, statistics._sum, data) + @unittest.skipIf(True, "temporarily disabled: see #25928") def test_decimal_snan_raises(self): # Adding sNAN should raise InvalidOperation. sNAN = Decimal('sNAN') -- cgit v0.12 From 150660eefd90e8004beb034ce885befed44df309 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Tue, 29 Dec 2015 02:15:35 +0100 Subject: Issue #25940: Make buildbots usable again until a solution is found. --- Lib/test/test_ssl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 548feeb..d1a9088 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1342,6 +1342,7 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) +@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1711,6 +1712,7 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret + @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet("svn.python.org"): sock = socket.socket(socket.AF_INET) -- cgit v0.12 From 3c61a448f106c7cafb050ff7be5c5c421273e68f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:17:58 -0800 Subject: Issue #24725: Skip the test_socket.testFDPassEmpty on OS X. In OS X 10.11, the test fails consistently due to a platform change since 10.10. Thanks to Jeff Ramnani for the patch. --- Lib/test/test_socket.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 1e355ea..ad1efb2 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,6 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) + @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v0.12 From 3bbad12b5bd41235bfb6e675606b046d30296eed Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:21:44 -0800 Subject: Tweak skipping message --- Lib/test/test_socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index ad1efb2..99707ce 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2809,7 +2809,7 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase): nbytes = self.sendmsgToServer([msg]) self.assertEqual(nbytes, len(msg)) - @unittest.skipIf(sys.platform == "darwin", "skipping, see issue #24725") + @unittest.skipIf(sys.platform == "darwin", "see issue #24725") def testFDPassEmpty(self): # Try to pass an empty FD array. Can receive either no array # or an empty array. -- cgit v0.12 From 838629a1fea6ec30342522dc0b087fc17872f1f0 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:23:35 -0800 Subject: Issue #25234: Skip test_eintr.test_os_open under OS X. Test inconsistently hangs. --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 531d576..e1ed3da 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -345,6 +345,7 @@ class SocketEINTRTest(EINTRBaseTest): fd = os.open(path, os.O_WRONLY) os.close(fd) + @unittest.skipIf(sys.platform == "darwin", "hangs under OS X; see issue #25234") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) -- cgit v0.12 From 050391774a4d1233c392f4e113e51c5a44b3e0ba Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:28:19 -0800 Subject: Issue #25930: Document that os.unlink and os.remove are *semantically* identical. Saying that the functions were identical confused some users who were upset when the functions were no longer simply the same function under different names. Thanks to Anthony Sottile for the bug report and Swati Jaiswal for the initial patch. --- Doc/library/os.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index c1193ad..373cafc 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1789,7 +1789,7 @@ features: be raised; on Unix, the directory entry is removed but the storage allocated to the file is not made available until the original file is no longer in use. - This function is identical to :func:`unlink`. + This function is semantically identical to :func:`unlink`. .. versionadded:: 3.3 The *dir_fd* argument. @@ -2452,10 +2452,10 @@ features: .. function:: unlink(path, *, dir_fd=None) - Remove (delete) the file *path*. This function is identical to - :func:`remove`; the ``unlink`` name is its traditional Unix - name. Please see the documentation for :func:`remove` for - further information. + Remove (delete) the file *path*. This function is semantically + identical to :func:`remove`; the ``unlink`` name is its + traditional Unix name. Please see the documentation for + :func:`remove` for further information. .. versionadded:: 3.3 The *dir_fd* parameter. -- cgit v0.12 From eae30790417d7113599e21639815c1e00798b9ed Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 28 Dec 2015 17:55:27 -0800 Subject: Issue #25802: Deprecate load_module() on importlib.machinery.SourceFileLoader and SourcelessFileLoader. They were the only remaining implementations of load_module() not documented as deprecated. --- Doc/library/importlib.rst | 8 + Doc/whatsnew/3.6.rst | 7 +- Lib/importlib/_bootstrap_external.py | 1 + Misc/NEWS | 3 + Python/importlib_external.h | 3033 +++++++++++++++++----------------- 5 files changed, 1535 insertions(+), 1517 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 02b0a11..d800835 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -921,6 +921,10 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.Loader.load_module` where specifying the name of the module to load is optional. + .. deprecated:: 3.6 + + Use :meth:`importlib.abc.Loader.exec_module` instead. + .. class:: SourcelessFileLoader(fullname, path) @@ -960,6 +964,10 @@ find and load modules. Concrete implementation of :meth:`importlib.abc.Loader.load_module` where specifying the name of the module to load is optional. + .. deprecated:: 3.6 + + Use :meth:`importlib.abc.Loader.exec_module` instead. + .. class:: ExtensionFileLoader(fullname, path) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c660933..bf5161d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -189,7 +189,12 @@ become proper keywords in Python 3.7. Deprecated Python modules, functions and methods ------------------------------------------------ -* None yet. +* :meth:`importlib.machinery.SourceFileLoader` and + :meth:`importlib.machinery.SourcelessFileLoader` are now deprecated. They + were the only remaining implementations of + :meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not + been deprecated in previous versions of Python in favour of + :meth:`importlib.abc.Loader.exec_module`. Deprecated functions and types of the C API diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index c58a62a..f274501 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -655,6 +655,7 @@ class _LoaderBasics: _bootstrap._call_with_frames_removed(exec, code, module.__dict__) def load_module(self, fullname): + """This module is deprecated.""" return _bootstrap._load_module_shim(self, fullname) diff --git a/Misc/NEWS b/Misc/NEWS index b042090..bf3ee24 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -123,6 +123,9 @@ Core and Builtins Library ------- +- Issue #25802: Document as deprecated the remaining implementations of + importlib.abc.Loader.load_module(). + - Issue #25928: Add Decimal.as_integer_ratio(). - Issue #25447: Copying the lru_cache() wrapper object now always works, diff --git a/Python/importlib_external.h b/Python/importlib_external.h index dd322a9..5828ceb 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1039,581 +1039,304 @@ const unsigned char _Py_M__importlib_external[] = { 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, 0,0,3,0,0,0,67,0,0,0,115,16,0,0,0,116, 0,0,106,1,0,124,0,0,124,1,0,131,2,0,83,41, - 1,78,41,2,114,114,0,0,0,218,17,95,108,111,97,100, - 95,109,111,100,117,108,101,95,115,104,105,109,41,2,114,100, - 0,0,0,114,119,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,11,108,111,97,100,95,109,111, - 100,117,108,101,145,2,0,0,115,2,0,0,0,0,1,122, - 25,95,76,111,97,100,101,114,66,97,115,105,99,115,46,108, - 111,97,100,95,109,111,100,117,108,101,78,41,8,114,105,0, - 0,0,114,104,0,0,0,114,106,0,0,0,114,107,0,0, - 0,114,153,0,0,0,114,180,0,0,0,114,185,0,0,0, - 114,187,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,178,0,0,0,121,2, - 0,0,115,10,0,0,0,12,3,6,2,12,8,12,3,12, - 8,114,178,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,4,0,0,0,64,0,0,0,115,106,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,100, - 2,0,132,0,0,90,3,0,100,3,0,100,4,0,132,0, - 0,90,4,0,100,5,0,100,6,0,132,0,0,90,5,0, - 100,7,0,100,8,0,132,0,0,90,6,0,100,9,0,100, - 10,0,132,0,0,90,7,0,100,11,0,100,18,0,100,13, - 0,100,14,0,132,0,1,90,8,0,100,15,0,100,16,0, - 132,0,0,90,9,0,100,17,0,83,41,19,218,12,83,111, - 117,114,99,101,76,111,97,100,101,114,99,2,0,0,0,0, - 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, - 10,0,0,0,116,0,0,130,1,0,100,1,0,83,41,2, - 122,178,79,112,116,105,111,110,97,108,32,109,101,116,104,111, - 100,32,116,104,97,116,32,114,101,116,117,114,110,115,32,116, - 104,101,32,109,111,100,105,102,105,99,97,116,105,111,110,32, - 116,105,109,101,32,40,97,110,32,105,110,116,41,32,102,111, - 114,32,116,104,101,10,32,32,32,32,32,32,32,32,115,112, - 101,99,105,102,105,101,100,32,112,97,116,104,44,32,119,104, - 101,114,101,32,112,97,116,104,32,105,115,32,97,32,115,116, - 114,46,10,10,32,32,32,32,32,32,32,32,82,97,105,115, + 1,122,26,84,104,105,115,32,109,111,100,117,108,101,32,105, + 115,32,100,101,112,114,101,99,97,116,101,100,46,41,2,114, + 114,0,0,0,218,17,95,108,111,97,100,95,109,111,100,117, + 108,101,95,115,104,105,109,41,2,114,100,0,0,0,114,119, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,145, + 2,0,0,115,2,0,0,0,0,2,122,25,95,76,111,97, + 100,101,114,66,97,115,105,99,115,46,108,111,97,100,95,109, + 111,100,117,108,101,78,41,8,114,105,0,0,0,114,104,0, + 0,0,114,106,0,0,0,114,107,0,0,0,114,153,0,0, + 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,178,0,0,0,121,2,0,0,115,10,0, + 0,0,12,3,6,2,12,8,12,3,12,8,114,178,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, + 0,0,64,0,0,0,115,106,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,100,2,0,132,0,0, + 90,3,0,100,3,0,100,4,0,132,0,0,90,4,0,100, + 5,0,100,6,0,132,0,0,90,5,0,100,7,0,100,8, + 0,132,0,0,90,6,0,100,9,0,100,10,0,132,0,0, + 90,7,0,100,11,0,100,18,0,100,13,0,100,14,0,132, + 0,1,90,8,0,100,15,0,100,16,0,132,0,0,90,9, + 0,100,17,0,83,41,19,218,12,83,111,117,114,99,101,76, + 111,97,100,101,114,99,2,0,0,0,0,0,0,0,2,0, + 0,0,1,0,0,0,67,0,0,0,115,10,0,0,0,116, + 0,0,130,1,0,100,1,0,83,41,2,122,178,79,112,116, + 105,111,110,97,108,32,109,101,116,104,111,100,32,116,104,97, + 116,32,114,101,116,117,114,110,115,32,116,104,101,32,109,111, + 100,105,102,105,99,97,116,105,111,110,32,116,105,109,101,32, + 40,97,110,32,105,110,116,41,32,102,111,114,32,116,104,101, + 10,32,32,32,32,32,32,32,32,115,112,101,99,105,102,105, + 101,100,32,112,97,116,104,44,32,119,104,101,114,101,32,112, + 97,116,104,32,105,115,32,97,32,115,116,114,46,10,10,32, + 32,32,32,32,32,32,32,82,97,105,115,101,115,32,73,79, + 69,114,114,111,114,32,119,104,101,110,32,116,104,101,32,112, + 97,116,104,32,99,97,110,110,111,116,32,98,101,32,104,97, + 110,100,108,101,100,46,10,32,32,32,32,32,32,32,32,78, + 41,1,218,7,73,79,69,114,114,111,114,41,2,114,100,0, + 0,0,114,35,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,10,112,97,116,104,95,109,116,105, + 109,101,152,2,0,0,115,2,0,0,0,0,6,122,23,83, + 111,117,114,99,101,76,111,97,100,101,114,46,112,97,116,104, + 95,109,116,105,109,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,67,0,0,0,115,19,0,0,0, + 100,1,0,124,0,0,106,0,0,124,1,0,131,1,0,105, + 1,0,83,41,2,97,170,1,0,0,79,112,116,105,111,110, + 97,108,32,109,101,116,104,111,100,32,114,101,116,117,114,110, + 105,110,103,32,97,32,109,101,116,97,100,97,116,97,32,100, + 105,99,116,32,102,111,114,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,112,97,116,104,10,32,32,32,32,32, + 32,32,32,116,111,32,98,121,32,116,104,101,32,112,97,116, + 104,32,40,115,116,114,41,46,10,32,32,32,32,32,32,32, + 32,80,111,115,115,105,98,108,101,32,107,101,121,115,58,10, + 32,32,32,32,32,32,32,32,45,32,39,109,116,105,109,101, + 39,32,40,109,97,110,100,97,116,111,114,121,41,32,105,115, + 32,116,104,101,32,110,117,109,101,114,105,99,32,116,105,109, + 101,115,116,97,109,112,32,111,102,32,108,97,115,116,32,115, + 111,117,114,99,101,10,32,32,32,32,32,32,32,32,32,32, + 99,111,100,101,32,109,111,100,105,102,105,99,97,116,105,111, + 110,59,10,32,32,32,32,32,32,32,32,45,32,39,115,105, + 122,101,39,32,40,111,112,116,105,111,110,97,108,41,32,105, + 115,32,116,104,101,32,115,105,122,101,32,105,110,32,98,121, + 116,101,115,32,111,102,32,116,104,101,32,115,111,117,114,99, + 101,32,99,111,100,101,46,10,10,32,32,32,32,32,32,32, + 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, + 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, + 32,116,104,101,32,108,111,97,100,101,114,32,116,111,32,114, + 101,97,100,32,98,121,116,101,99,111,100,101,32,102,105,108, + 101,115,46,10,32,32,32,32,32,32,32,32,82,97,105,115, 101,115,32,73,79,69,114,114,111,114,32,119,104,101,110,32, 116,104,101,32,112,97,116,104,32,99,97,110,110,111,116,32, 98,101,32,104,97,110,100,108,101,100,46,10,32,32,32,32, - 32,32,32,32,78,41,1,218,7,73,79,69,114,114,111,114, + 32,32,32,32,114,126,0,0,0,41,1,114,190,0,0,0, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,109,116,105,109,101,151,2,0,0,115,2,0,0,0, - 0,6,122,23,83,111,117,114,99,101,76,111,97,100,101,114, - 46,112,97,116,104,95,109,116,105,109,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, - 115,19,0,0,0,100,1,0,124,0,0,106,0,0,124,1, - 0,131,1,0,105,1,0,83,41,2,97,170,1,0,0,79, - 112,116,105,111,110,97,108,32,109,101,116,104,111,100,32,114, - 101,116,117,114,110,105,110,103,32,97,32,109,101,116,97,100, - 97,116,97,32,100,105,99,116,32,102,111,114,32,116,104,101, - 32,115,112,101,99,105,102,105,101,100,32,112,97,116,104,10, - 32,32,32,32,32,32,32,32,116,111,32,98,121,32,116,104, - 101,32,112,97,116,104,32,40,115,116,114,41,46,10,32,32, - 32,32,32,32,32,32,80,111,115,115,105,98,108,101,32,107, - 101,121,115,58,10,32,32,32,32,32,32,32,32,45,32,39, - 109,116,105,109,101,39,32,40,109,97,110,100,97,116,111,114, - 121,41,32,105,115,32,116,104,101,32,110,117,109,101,114,105, - 99,32,116,105,109,101,115,116,97,109,112,32,111,102,32,108, - 97,115,116,32,115,111,117,114,99,101,10,32,32,32,32,32, - 32,32,32,32,32,99,111,100,101,32,109,111,100,105,102,105, - 99,97,116,105,111,110,59,10,32,32,32,32,32,32,32,32, - 45,32,39,115,105,122,101,39,32,40,111,112,116,105,111,110, - 97,108,41,32,105,115,32,116,104,101,32,115,105,122,101,32, - 105,110,32,98,121,116,101,115,32,111,102,32,116,104,101,32, - 115,111,117,114,99,101,32,99,111,100,101,46,10,10,32,32, - 32,32,32,32,32,32,73,109,112,108,101,109,101,110,116,105, - 110,103,32,116,104,105,115,32,109,101,116,104,111,100,32,97, - 108,108,111,119,115,32,116,104,101,32,108,111,97,100,101,114, - 32,116,111,32,114,101,97,100,32,98,121,116,101,99,111,100, - 101,32,102,105,108,101,115,46,10,32,32,32,32,32,32,32, - 32,82,97,105,115,101,115,32,73,79,69,114,114,111,114,32, - 119,104,101,110,32,116,104,101,32,112,97,116,104,32,99,97, - 110,110,111,116,32,98,101,32,104,97,110,100,108,101,100,46, - 10,32,32,32,32,32,32,32,32,114,126,0,0,0,41,1, - 114,190,0,0,0,41,2,114,100,0,0,0,114,35,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,10,112,97,116,104,95,115,116,97,116,115,159,2,0,0, - 115,2,0,0,0,0,11,122,23,83,111,117,114,99,101,76, - 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, - 99,4,0,0,0,0,0,0,0,4,0,0,0,3,0,0, - 0,67,0,0,0,115,16,0,0,0,124,0,0,106,0,0, - 124,2,0,124,3,0,131,2,0,83,41,1,122,228,79,112, - 116,105,111,110,97,108,32,109,101,116,104,111,100,32,119,104, - 105,99,104,32,119,114,105,116,101,115,32,100,97,116,97,32, - 40,98,121,116,101,115,41,32,116,111,32,97,32,102,105,108, - 101,32,112,97,116,104,32,40,97,32,115,116,114,41,46,10, - 10,32,32,32,32,32,32,32,32,73,109,112,108,101,109,101, - 110,116,105,110,103,32,116,104,105,115,32,109,101,116,104,111, - 100,32,97,108,108,111,119,115,32,102,111,114,32,116,104,101, - 32,119,114,105,116,105,110,103,32,111,102,32,98,121,116,101, - 99,111,100,101,32,102,105,108,101,115,46,10,10,32,32,32, - 32,32,32,32,32,84,104,101,32,115,111,117,114,99,101,32, - 112,97,116,104,32,105,115,32,110,101,101,100,101,100,32,105, - 110,32,111,114,100,101,114,32,116,111,32,99,111,114,114,101, - 99,116,108,121,32,116,114,97,110,115,102,101,114,32,112,101, - 114,109,105,115,115,105,111,110,115,10,32,32,32,32,32,32, - 32,32,41,1,218,8,115,101,116,95,100,97,116,97,41,4, - 114,100,0,0,0,114,90,0,0,0,90,10,99,97,99,104, - 101,95,112,97,116,104,114,53,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,15,95,99,97,99, - 104,101,95,98,121,116,101,99,111,100,101,172,2,0,0,115, - 2,0,0,0,0,8,122,28,83,111,117,114,99,101,76,111, - 97,100,101,114,46,95,99,97,99,104,101,95,98,121,116,101, - 99,111,100,101,99,3,0,0,0,0,0,0,0,3,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,122,150,79,112,116,105,111,110,97,108,32,109, - 101,116,104,111,100,32,119,104,105,99,104,32,119,114,105,116, - 101,115,32,100,97,116,97,32,40,98,121,116,101,115,41,32, - 116,111,32,97,32,102,105,108,101,32,112,97,116,104,32,40, - 97,32,115,116,114,41,46,10,10,32,32,32,32,32,32,32, - 32,73,109,112,108,101,109,101,110,116,105,110,103,32,116,104, - 105,115,32,109,101,116,104,111,100,32,97,108,108,111,119,115, - 32,102,111,114,32,116,104,101,32,119,114,105,116,105,110,103, - 32,111,102,32,98,121,116,101,99,111,100,101,32,102,105,108, - 101,115,46,10,32,32,32,32,32,32,32,32,78,114,4,0, - 0,0,41,3,114,100,0,0,0,114,35,0,0,0,114,53, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,192,0,0,0,182,2,0,0,115,0,0,0,0, - 122,21,83,111,117,114,99,101,76,111,97,100,101,114,46,115, - 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, - 5,0,0,0,16,0,0,0,67,0,0,0,115,105,0,0, - 0,124,0,0,106,0,0,124,1,0,131,1,0,125,2,0, - 121,19,0,124,0,0,106,1,0,124,2,0,131,1,0,125, - 3,0,87,110,58,0,4,116,2,0,107,10,0,114,94,0, - 1,125,4,0,1,122,26,0,116,3,0,100,1,0,100,2, - 0,124,1,0,131,1,1,124,4,0,130,2,0,87,89,100, - 3,0,100,3,0,125,4,0,126,4,0,88,110,1,0,88, - 116,4,0,124,3,0,131,1,0,83,41,4,122,52,67,111, - 110,99,114,101,116,101,32,105,109,112,108,101,109,101,110,116, - 97,116,105,111,110,32,111,102,32,73,110,115,112,101,99,116, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,46,122,39,115,111,117,114,99,101,32,110,111,116,32,97, - 118,97,105,108,97,98,108,101,32,116,104,114,111,117,103,104, - 32,103,101,116,95,100,97,116,97,40,41,114,98,0,0,0, - 78,41,5,114,151,0,0,0,218,8,103,101,116,95,100,97, - 116,97,114,40,0,0,0,114,99,0,0,0,114,149,0,0, - 0,41,5,114,100,0,0,0,114,119,0,0,0,114,35,0, - 0,0,114,147,0,0,0,218,3,101,120,99,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,10,103,101,116, - 95,115,111,117,114,99,101,189,2,0,0,115,14,0,0,0, - 0,2,15,1,3,1,19,1,18,1,9,1,31,1,122,23, - 83,111,117,114,99,101,76,111,97,100,101,114,46,103,101,116, - 95,115,111,117,114,99,101,218,9,95,111,112,116,105,109,105, - 122,101,114,29,0,0,0,99,3,0,0,0,1,0,0,0, - 4,0,0,0,9,0,0,0,67,0,0,0,115,34,0,0, - 0,116,0,0,106,1,0,116,2,0,124,1,0,124,2,0, - 100,1,0,100,2,0,100,3,0,100,4,0,124,3,0,131, - 4,2,83,41,5,122,130,82,101,116,117,114,110,32,116,104, - 101,32,99,111,100,101,32,111,98,106,101,99,116,32,99,111, - 109,112,105,108,101,100,32,102,114,111,109,32,115,111,117,114, - 99,101,46,10,10,32,32,32,32,32,32,32,32,84,104,101, - 32,39,100,97,116,97,39,32,97,114,103,117,109,101,110,116, - 32,99,97,110,32,98,101,32,97,110,121,32,111,98,106,101, - 99,116,32,116,121,112,101,32,116,104,97,116,32,99,111,109, - 112,105,108,101,40,41,32,115,117,112,112,111,114,116,115,46, - 10,32,32,32,32,32,32,32,32,114,183,0,0,0,218,12, - 100,111,110,116,95,105,110,104,101,114,105,116,84,114,68,0, - 0,0,41,3,114,114,0,0,0,114,182,0,0,0,218,7, - 99,111,109,112,105,108,101,41,4,114,100,0,0,0,114,53, - 0,0,0,114,35,0,0,0,114,197,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,14,115,111, - 117,114,99,101,95,116,111,95,99,111,100,101,199,2,0,0, - 115,4,0,0,0,0,5,21,1,122,27,83,111,117,114,99, - 101,76,111,97,100,101,114,46,115,111,117,114,99,101,95,116, - 111,95,99,111,100,101,99,2,0,0,0,0,0,0,0,10, - 0,0,0,43,0,0,0,67,0,0,0,115,183,1,0,0, - 124,0,0,106,0,0,124,1,0,131,1,0,125,2,0,100, - 1,0,125,3,0,121,16,0,116,1,0,124,2,0,131,1, - 0,125,4,0,87,110,24,0,4,116,2,0,107,10,0,114, - 63,0,1,1,1,100,1,0,125,4,0,89,110,205,0,88, - 121,19,0,124,0,0,106,3,0,124,2,0,131,1,0,125, - 5,0,87,110,18,0,4,116,4,0,107,10,0,114,103,0, - 1,1,1,89,110,165,0,88,116,5,0,124,5,0,100,2, - 0,25,131,1,0,125,3,0,121,19,0,124,0,0,106,6, - 0,124,4,0,131,1,0,125,6,0,87,110,18,0,4,116, - 7,0,107,10,0,114,159,0,1,1,1,89,110,109,0,88, - 121,34,0,116,8,0,124,6,0,100,3,0,124,5,0,100, - 4,0,124,1,0,100,5,0,124,4,0,131,1,3,125,7, - 0,87,110,24,0,4,116,9,0,116,10,0,102,2,0,107, - 10,0,114,220,0,1,1,1,89,110,48,0,88,116,11,0, - 106,12,0,100,6,0,124,4,0,124,2,0,131,3,0,1, - 116,13,0,124,7,0,100,4,0,124,1,0,100,7,0,124, - 4,0,100,8,0,124,2,0,131,1,3,83,124,0,0,106, - 6,0,124,2,0,131,1,0,125,8,0,124,0,0,106,14, - 0,124,8,0,124,2,0,131,2,0,125,9,0,116,11,0, - 106,12,0,100,9,0,124,2,0,131,2,0,1,116,15,0, - 106,16,0,12,114,179,1,124,4,0,100,1,0,107,9,0, - 114,179,1,124,3,0,100,1,0,107,9,0,114,179,1,116, - 17,0,124,9,0,124,3,0,116,18,0,124,8,0,131,1, - 0,131,3,0,125,6,0,121,39,0,124,0,0,106,19,0, - 124,2,0,124,4,0,124,6,0,131,3,0,1,116,11,0, - 106,12,0,100,10,0,124,4,0,131,2,0,1,87,110,18, - 0,4,116,2,0,107,10,0,114,178,1,1,1,1,89,110, - 1,0,88,124,9,0,83,41,11,122,190,67,111,110,99,114, - 101,116,101,32,105,109,112,108,101,109,101,110,116,97,116,105, - 111,110,32,111,102,32,73,110,115,112,101,99,116,76,111,97, - 100,101,114,46,103,101,116,95,99,111,100,101,46,10,10,32, - 32,32,32,32,32,32,32,82,101,97,100,105,110,103,32,111, - 102,32,98,121,116,101,99,111,100,101,32,114,101,113,117,105, - 114,101,115,32,112,97,116,104,95,115,116,97,116,115,32,116, - 111,32,98,101,32,105,109,112,108,101,109,101,110,116,101,100, - 46,32,84,111,32,119,114,105,116,101,10,32,32,32,32,32, - 32,32,32,98,121,116,101,99,111,100,101,44,32,115,101,116, - 95,100,97,116,97,32,109,117,115,116,32,97,108,115,111,32, - 98,101,32,105,109,112,108,101,109,101,110,116,101,100,46,10, - 10,32,32,32,32,32,32,32,32,78,114,126,0,0,0,114, - 132,0,0,0,114,98,0,0,0,114,35,0,0,0,122,13, - 123,125,32,109,97,116,99,104,101,115,32,123,125,114,89,0, - 0,0,114,90,0,0,0,122,19,99,111,100,101,32,111,98, - 106,101,99,116,32,102,114,111,109,32,123,125,122,10,119,114, - 111,116,101,32,123,33,114,125,41,20,114,151,0,0,0,114, - 79,0,0,0,114,66,0,0,0,114,191,0,0,0,114,189, - 0,0,0,114,14,0,0,0,114,194,0,0,0,114,40,0, - 0,0,114,135,0,0,0,114,99,0,0,0,114,130,0,0, - 0,114,114,0,0,0,114,129,0,0,0,114,141,0,0,0, - 114,200,0,0,0,114,7,0,0,0,218,19,100,111,110,116, - 95,119,114,105,116,101,95,98,121,116,101,99,111,100,101,114, - 144,0,0,0,114,31,0,0,0,114,193,0,0,0,41,10, - 114,100,0,0,0,114,119,0,0,0,114,90,0,0,0,114, - 133,0,0,0,114,89,0,0,0,218,2,115,116,114,53,0, - 0,0,218,10,98,121,116,101,115,95,100,97,116,97,114,147, - 0,0,0,90,11,99,111,100,101,95,111,98,106,101,99,116, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 181,0,0,0,207,2,0,0,115,78,0,0,0,0,7,15, - 1,6,1,3,1,16,1,13,1,11,2,3,1,19,1,13, - 1,5,2,16,1,3,1,19,1,13,1,5,2,3,1,9, - 1,12,1,13,1,19,1,5,2,12,1,7,1,15,1,6, - 1,7,1,15,1,18,1,16,1,22,1,12,1,9,1,15, - 1,3,1,19,1,20,1,13,1,5,1,122,21,83,111,117, - 114,99,101,76,111,97,100,101,114,46,103,101,116,95,99,111, - 100,101,78,114,87,0,0,0,41,10,114,105,0,0,0,114, - 104,0,0,0,114,106,0,0,0,114,190,0,0,0,114,191, - 0,0,0,114,193,0,0,0,114,192,0,0,0,114,196,0, - 0,0,114,200,0,0,0,114,181,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,188,0,0,0,149,2,0,0,115,14,0,0,0,12,2, - 12,8,12,13,12,10,12,7,12,10,18,8,114,188,0,0, - 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, - 0,0,0,0,0,0,115,112,0,0,0,101,0,0,90,1, - 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, - 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, - 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, - 0,101,7,0,135,0,0,102,1,0,100,8,0,100,9,0, - 134,0,0,131,1,0,90,8,0,101,7,0,100,10,0,100, - 11,0,132,0,0,131,1,0,90,9,0,100,12,0,100,13, - 0,132,0,0,90,10,0,135,0,0,83,41,14,218,10,70, - 105,108,101,76,111,97,100,101,114,122,103,66,97,115,101,32, - 102,105,108,101,32,108,111,97,100,101,114,32,99,108,97,115, - 115,32,119,104,105,99,104,32,105,109,112,108,101,109,101,110, - 116,115,32,116,104,101,32,108,111,97,100,101,114,32,112,114, - 111,116,111,99,111,108,32,109,101,116,104,111,100,115,32,116, - 104,97,116,10,32,32,32,32,114,101,113,117,105,114,101,32, - 102,105,108,101,32,115,121,115,116,101,109,32,117,115,97,103, - 101,46,99,3,0,0,0,0,0,0,0,3,0,0,0,2, - 0,0,0,67,0,0,0,115,22,0,0,0,124,1,0,124, - 0,0,95,0,0,124,2,0,124,0,0,95,1,0,100,1, - 0,83,41,2,122,75,67,97,99,104,101,32,116,104,101,32, - 109,111,100,117,108,101,32,110,97,109,101,32,97,110,100,32, - 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, - 102,105,108,101,32,102,111,117,110,100,32,98,121,32,116,104, - 101,10,32,32,32,32,32,32,32,32,102,105,110,100,101,114, - 46,78,41,2,114,98,0,0,0,114,35,0,0,0,41,3, - 114,100,0,0,0,114,119,0,0,0,114,35,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,179, - 0,0,0,8,3,0,0,115,4,0,0,0,0,3,9,1, - 122,19,70,105,108,101,76,111,97,100,101,114,46,95,95,105, - 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, - 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, - 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, - 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, - 83,41,1,78,41,2,218,9,95,95,99,108,97,115,115,95, - 95,114,111,0,0,0,41,2,114,100,0,0,0,218,5,111, - 116,104,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,95,101,113,95,95,14,3,0,0,115, - 4,0,0,0,0,1,18,1,122,17,70,105,108,101,76,111, - 97,100,101,114,46,95,95,101,113,95,95,99,1,0,0,0, - 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, - 115,26,0,0,0,116,0,0,124,0,0,106,1,0,131,1, - 0,116,0,0,124,0,0,106,2,0,131,1,0,65,83,41, - 1,78,41,3,218,4,104,97,115,104,114,98,0,0,0,114, - 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,218,8,95,95,104,97, - 115,104,95,95,18,3,0,0,115,2,0,0,0,0,1,122, - 19,70,105,108,101,76,111,97,100,101,114,46,95,95,104,97, - 115,104,95,95,99,2,0,0,0,0,0,0,0,2,0,0, - 0,3,0,0,0,3,0,0,0,115,22,0,0,0,116,0, - 0,116,1,0,124,0,0,131,2,0,106,2,0,124,1,0, - 131,1,0,83,41,1,122,100,76,111,97,100,32,97,32,109, - 111,100,117,108,101,32,102,114,111,109,32,97,32,102,105,108, - 101,46,10,10,32,32,32,32,32,32,32,32,84,104,105,115, - 32,109,101,116,104,111,100,32,105,115,32,100,101,112,114,101, - 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, - 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, - 100,46,10,10,32,32,32,32,32,32,32,32,41,3,218,5, - 115,117,112,101,114,114,204,0,0,0,114,187,0,0,0,41, - 2,114,100,0,0,0,114,119,0,0,0,41,1,114,205,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,187,0,0, - 0,21,3,0,0,115,2,0,0,0,0,10,122,22,70,105, - 108,101,76,111,97,100,101,114,46,108,111,97,100,95,109,111, - 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,7,0,0,0,124,0, - 0,106,0,0,83,41,1,122,58,82,101,116,117,114,110,32, - 116,104,101,32,112,97,116,104,32,116,111,32,116,104,101,32, - 115,111,117,114,99,101,32,102,105,108,101,32,97,115,32,102, - 111,117,110,100,32,98,121,32,116,104,101,32,102,105,110,100, - 101,114,46,41,1,114,35,0,0,0,41,2,114,100,0,0, - 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,151,0,0,0,33,3,0,0,115,2, - 0,0,0,0,3,122,23,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,102,105,108,101,110,97,109,101,99,2, - 0,0,0,0,0,0,0,3,0,0,0,9,0,0,0,67, - 0,0,0,115,42,0,0,0,116,0,0,106,1,0,124,1, - 0,100,1,0,131,2,0,143,17,0,125,2,0,124,2,0, - 106,2,0,131,0,0,83,87,100,2,0,81,82,88,100,2, - 0,83,41,3,122,39,82,101,116,117,114,110,32,116,104,101, - 32,100,97,116,97,32,102,114,111,109,32,112,97,116,104,32, - 97,115,32,114,97,119,32,98,121,116,101,115,46,218,1,114, - 78,41,3,114,49,0,0,0,114,50,0,0,0,90,4,114, - 101,97,100,41,3,114,100,0,0,0,114,35,0,0,0,114, - 54,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,194,0,0,0,38,3,0,0,115,4,0,0, - 0,0,2,21,1,122,19,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,100,97,116,97,41,11,114,105,0,0, - 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, - 114,179,0,0,0,114,207,0,0,0,114,209,0,0,0,114, - 116,0,0,0,114,187,0,0,0,114,151,0,0,0,114,194, - 0,0,0,114,4,0,0,0,114,4,0,0,0,41,1,114, - 205,0,0,0,114,5,0,0,0,114,204,0,0,0,3,3, - 0,0,115,14,0,0,0,12,3,6,2,12,6,12,4,12, - 3,24,12,18,5,114,204,0,0,0,99,0,0,0,0,0, - 0,0,0,0,0,0,0,4,0,0,0,64,0,0,0,115, - 64,0,0,0,101,0,0,90,1,0,100,0,0,90,2,0, - 100,1,0,90,3,0,100,2,0,100,3,0,132,0,0,90, - 4,0,100,4,0,100,5,0,132,0,0,90,5,0,100,6, - 0,100,7,0,100,8,0,100,9,0,132,0,1,90,6,0, - 100,10,0,83,41,11,218,16,83,111,117,114,99,101,70,105, - 108,101,76,111,97,100,101,114,122,62,67,111,110,99,114,101, - 116,101,32,105,109,112,108,101,109,101,110,116,97,116,105,111, - 110,32,111,102,32,83,111,117,114,99,101,76,111,97,100,101, - 114,32,117,115,105,110,103,32,116,104,101,32,102,105,108,101, - 32,115,121,115,116,101,109,46,99,2,0,0,0,0,0,0, - 0,3,0,0,0,4,0,0,0,67,0,0,0,115,34,0, - 0,0,116,0,0,124,1,0,131,1,0,125,2,0,100,1, - 0,124,2,0,106,1,0,100,2,0,124,2,0,106,2,0, - 105,2,0,83,41,3,122,33,82,101,116,117,114,110,32,116, - 104,101,32,109,101,116,97,100,97,116,97,32,102,111,114,32, - 116,104,101,32,112,97,116,104,46,114,126,0,0,0,114,127, - 0,0,0,41,3,114,39,0,0,0,218,8,115,116,95,109, - 116,105,109,101,90,7,115,116,95,115,105,122,101,41,3,114, - 100,0,0,0,114,35,0,0,0,114,202,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,191,0, - 0,0,48,3,0,0,115,4,0,0,0,0,2,12,1,122, - 27,83,111,117,114,99,101,70,105,108,101,76,111,97,100,101, - 114,46,112,97,116,104,95,115,116,97,116,115,99,4,0,0, - 0,0,0,0,0,5,0,0,0,5,0,0,0,67,0,0, - 0,115,34,0,0,0,116,0,0,124,1,0,131,1,0,125, - 4,0,124,0,0,106,1,0,124,2,0,124,3,0,100,1, - 0,124,4,0,131,2,1,83,41,2,78,218,5,95,109,111, - 100,101,41,2,114,97,0,0,0,114,192,0,0,0,41,5, - 114,100,0,0,0,114,90,0,0,0,114,89,0,0,0,114, - 53,0,0,0,114,42,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,193,0,0,0,53,3,0, - 0,115,4,0,0,0,0,2,12,1,122,32,83,111,117,114, - 99,101,70,105,108,101,76,111,97,100,101,114,46,95,99,97, - 99,104,101,95,98,121,116,101,99,111,100,101,114,214,0,0, - 0,105,182,1,0,0,99,3,0,0,0,1,0,0,0,9, - 0,0,0,17,0,0,0,67,0,0,0,115,62,1,0,0, - 116,0,0,124,1,0,131,1,0,92,2,0,125,4,0,125, - 5,0,103,0,0,125,6,0,120,54,0,124,4,0,114,80, - 0,116,1,0,124,4,0,131,1,0,12,114,80,0,116,0, - 0,124,4,0,131,1,0,92,2,0,125,4,0,125,7,0, - 124,6,0,106,2,0,124,7,0,131,1,0,1,113,27,0, - 87,120,135,0,116,3,0,124,6,0,131,1,0,68,93,121, - 0,125,7,0,116,4,0,124,4,0,124,7,0,131,2,0, - 125,4,0,121,17,0,116,5,0,106,6,0,124,4,0,131, - 1,0,1,87,113,94,0,4,116,7,0,107,10,0,114,155, - 0,1,1,1,119,94,0,89,113,94,0,4,116,8,0,107, - 10,0,114,214,0,1,125,8,0,1,122,28,0,116,9,0, - 106,10,0,100,1,0,124,4,0,124,8,0,131,3,0,1, - 100,2,0,83,87,89,100,2,0,100,2,0,125,8,0,126, - 8,0,88,113,94,0,88,113,94,0,87,121,36,0,116,11, - 0,124,1,0,124,2,0,124,3,0,131,3,0,1,116,9, - 0,106,10,0,100,3,0,124,1,0,131,2,0,1,87,110, - 56,0,4,116,8,0,107,10,0,114,57,1,1,125,8,0, - 1,122,24,0,116,9,0,106,10,0,100,1,0,124,1,0, - 124,8,0,131,3,0,1,87,89,100,2,0,100,2,0,125, - 8,0,126,8,0,88,110,1,0,88,100,2,0,83,41,4, - 122,27,87,114,105,116,101,32,98,121,116,101,115,32,100,97, - 116,97,32,116,111,32,97,32,102,105,108,101,46,122,27,99, - 111,117,108,100,32,110,111,116,32,99,114,101,97,116,101,32, - 123,33,114,125,58,32,123,33,114,125,78,122,12,99,114,101, - 97,116,101,100,32,123,33,114,125,41,12,114,38,0,0,0, - 114,46,0,0,0,114,157,0,0,0,114,33,0,0,0,114, - 28,0,0,0,114,3,0,0,0,90,5,109,107,100,105,114, - 218,15,70,105,108,101,69,120,105,115,116,115,69,114,114,111, - 114,114,40,0,0,0,114,114,0,0,0,114,129,0,0,0, - 114,55,0,0,0,41,9,114,100,0,0,0,114,35,0,0, - 0,114,53,0,0,0,114,214,0,0,0,218,6,112,97,114, - 101,110,116,114,94,0,0,0,114,27,0,0,0,114,23,0, - 0,0,114,195,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,192,0,0,0,58,3,0,0,115, - 42,0,0,0,0,2,18,1,6,2,22,1,18,1,17,2, - 19,1,15,1,3,1,17,1,13,2,7,1,18,3,9,1, - 10,1,27,1,3,1,16,1,20,1,18,2,12,1,122,25, - 83,111,117,114,99,101,70,105,108,101,76,111,97,100,101,114, - 46,115,101,116,95,100,97,116,97,78,41,7,114,105,0,0, - 0,114,104,0,0,0,114,106,0,0,0,114,107,0,0,0, - 114,191,0,0,0,114,193,0,0,0,114,192,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,212,0,0,0,44,3,0,0,115,8,0,0, - 0,12,2,6,2,12,5,12,5,114,212,0,0,0,99,0, - 0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,64, - 0,0,0,115,46,0,0,0,101,0,0,90,1,0,100,0, - 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, - 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, - 5,0,100,6,0,83,41,7,218,20,83,111,117,114,99,101, - 108,101,115,115,70,105,108,101,76,111,97,100,101,114,122,45, - 76,111,97,100,101,114,32,119,104,105,99,104,32,104,97,110, - 100,108,101,115,32,115,111,117,114,99,101,108,101,115,115,32, - 102,105,108,101,32,105,109,112,111,114,116,115,46,99,2,0, - 0,0,0,0,0,0,5,0,0,0,6,0,0,0,67,0, - 0,0,115,76,0,0,0,124,0,0,106,0,0,124,1,0, - 131,1,0,125,2,0,124,0,0,106,1,0,124,2,0,131, - 1,0,125,3,0,116,2,0,124,3,0,100,1,0,124,1, - 0,100,2,0,124,2,0,131,1,2,125,4,0,116,3,0, - 124,4,0,100,1,0,124,1,0,100,3,0,124,2,0,131, - 1,2,83,41,4,78,114,98,0,0,0,114,35,0,0,0, - 114,89,0,0,0,41,4,114,151,0,0,0,114,194,0,0, - 0,114,135,0,0,0,114,141,0,0,0,41,5,114,100,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,53,0,0, - 0,114,203,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,93,3,0,0,115,8, - 0,0,0,0,1,15,1,15,1,24,1,122,29,83,111,117, - 114,99,101,108,101,115,115,70,105,108,101,76,111,97,100,101, - 114,46,103,101,116,95,99,111,100,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, - 4,0,0,0,100,1,0,83,41,2,122,39,82,101,116,117, - 114,110,32,78,111,110,101,32,97,115,32,116,104,101,114,101, - 32,105,115,32,110,111,32,115,111,117,114,99,101,32,99,111, - 100,101,46,78,114,4,0,0,0,41,2,114,100,0,0,0, - 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,99,3,0,0,115,2,0, - 0,0,0,2,122,31,83,111,117,114,99,101,108,101,115,115, - 70,105,108,101,76,111,97,100,101,114,46,103,101,116,95,115, - 111,117,114,99,101,78,41,6,114,105,0,0,0,114,104,0, - 0,0,114,106,0,0,0,114,107,0,0,0,114,181,0,0, - 0,114,196,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,217,0,0,0,89, - 3,0,0,115,6,0,0,0,12,2,6,2,12,6,114,217, - 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, - 3,0,0,0,64,0,0,0,115,136,0,0,0,101,0,0, - 90,1,0,100,0,0,90,2,0,100,1,0,90,3,0,100, - 2,0,100,3,0,132,0,0,90,4,0,100,4,0,100,5, - 0,132,0,0,90,5,0,100,6,0,100,7,0,132,0,0, - 90,6,0,100,8,0,100,9,0,132,0,0,90,7,0,100, - 10,0,100,11,0,132,0,0,90,8,0,100,12,0,100,13, - 0,132,0,0,90,9,0,100,14,0,100,15,0,132,0,0, - 90,10,0,100,16,0,100,17,0,132,0,0,90,11,0,101, - 12,0,100,18,0,100,19,0,132,0,0,131,1,0,90,13, - 0,100,20,0,83,41,21,218,19,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,122,93,76,111, - 97,100,101,114,32,102,111,114,32,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,115,46,10,10,32,32,32, - 32,84,104,101,32,99,111,110,115,116,114,117,99,116,111,114, - 32,105,115,32,100,101,115,105,103,110,101,100,32,116,111,32, - 119,111,114,107,32,119,105,116,104,32,70,105,108,101,70,105, - 110,100,101,114,46,10,10,32,32,32,32,99,3,0,0,0, - 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, - 115,22,0,0,0,124,1,0,124,0,0,95,0,0,124,2, - 0,124,0,0,95,1,0,100,0,0,83,41,1,78,41,2, - 114,98,0,0,0,114,35,0,0,0,41,3,114,100,0,0, - 0,114,98,0,0,0,114,35,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,179,0,0,0,116, - 3,0,0,115,4,0,0,0,0,1,9,1,122,28,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,105,110,105,116,95,95,99,2,0,0,0,0, - 0,0,0,2,0,0,0,2,0,0,0,67,0,0,0,115, - 34,0,0,0,124,0,0,106,0,0,124,1,0,106,0,0, - 107,2,0,111,33,0,124,0,0,106,1,0,124,1,0,106, - 1,0,107,2,0,83,41,1,78,41,2,114,205,0,0,0, - 114,111,0,0,0,41,2,114,100,0,0,0,114,206,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,207,0,0,0,120,3,0,0,115,4,0,0,0,0,1, - 18,1,122,26,69,120,116,101,110,115,105,111,110,70,105,108, - 101,76,111,97,100,101,114,46,95,95,101,113,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, - 0,0,0,115,26,0,0,0,116,0,0,124,0,0,106,1, - 0,131,1,0,116,0,0,124,0,0,106,2,0,131,1,0, - 65,83,41,1,78,41,3,114,208,0,0,0,114,98,0,0, - 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,209,0,0, - 0,124,3,0,0,115,2,0,0,0,0,1,122,28,69,120, - 116,101,110,115,105,111,110,70,105,108,101,76,111,97,100,101, - 114,46,95,95,104,97,115,104,95,95,99,2,0,0,0,0, - 0,0,0,3,0,0,0,4,0,0,0,67,0,0,0,115, - 50,0,0,0,116,0,0,106,1,0,116,2,0,106,3,0, - 124,1,0,131,2,0,125,2,0,116,0,0,106,4,0,100, - 1,0,124,1,0,106,5,0,124,0,0,106,6,0,131,3, - 0,1,124,2,0,83,41,2,122,38,67,114,101,97,116,101, - 32,97,110,32,117,110,105,116,105,97,108,105,122,101,100,32, - 101,120,116,101,110,115,105,111,110,32,109,111,100,117,108,101, - 122,38,101,120,116,101,110,115,105,111,110,32,109,111,100,117, - 108,101,32,123,33,114,125,32,108,111,97,100,101,100,32,102, - 114,111,109,32,123,33,114,125,41,7,114,114,0,0,0,114, - 182,0,0,0,114,139,0,0,0,90,14,99,114,101,97,116, - 101,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, - 0,0,0,114,35,0,0,0,41,3,114,100,0,0,0,114, - 158,0,0,0,114,184,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,127,3,0, - 0,115,10,0,0,0,0,2,6,1,15,1,9,1,16,1, - 122,33,69,120,116,101,110,115,105,111,110,70,105,108,101,76, - 111,97,100,101,114,46,99,114,101,97,116,101,95,109,111,100, - 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 4,0,0,0,67,0,0,0,115,48,0,0,0,116,0,0, - 106,1,0,116,2,0,106,3,0,124,1,0,131,2,0,1, - 116,0,0,106,4,0,100,1,0,124,0,0,106,5,0,124, - 0,0,106,6,0,131,3,0,1,100,2,0,83,41,3,122, - 30,73,110,105,116,105,97,108,105,122,101,32,97,110,32,101, - 120,116,101,110,115,105,111,110,32,109,111,100,117,108,101,122, - 40,101,120,116,101,110,115,105,111,110,32,109,111,100,117,108, - 101,32,123,33,114,125,32,101,120,101,99,117,116,101,100,32, - 102,114,111,109,32,123,33,114,125,78,41,7,114,114,0,0, - 0,114,182,0,0,0,114,139,0,0,0,90,12,101,120,101, - 99,95,100,121,110,97,109,105,99,114,129,0,0,0,114,98, - 0,0,0,114,35,0,0,0,41,2,114,100,0,0,0,114, - 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,185,0,0,0,135,3,0,0,115,6,0,0, - 0,0,2,19,1,9,1,122,31,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,101,120,101, - 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, - 0,2,0,0,0,4,0,0,0,3,0,0,0,115,48,0, - 0,0,116,0,0,124,0,0,106,1,0,131,1,0,100,1, - 0,25,137,0,0,116,2,0,135,0,0,102,1,0,100,2, - 0,100,3,0,134,0,0,116,3,0,68,131,1,0,131,1, - 0,83,41,4,122,49,82,101,116,117,114,110,32,84,114,117, - 101,32,105,102,32,116,104,101,32,101,120,116,101,110,115,105, - 111,110,32,109,111,100,117,108,101,32,105,115,32,97,32,112, - 97,99,107,97,103,101,46,114,29,0,0,0,99,1,0,0, - 0,0,0,0,0,2,0,0,0,4,0,0,0,51,0,0, - 0,115,31,0,0,0,124,0,0,93,21,0,125,1,0,136, - 0,0,100,0,0,124,1,0,23,107,2,0,86,1,113,3, - 0,100,1,0,83,41,2,114,179,0,0,0,78,114,4,0, - 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, - 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,4, - 0,0,0,114,5,0,0,0,250,9,60,103,101,110,101,120, - 112,114,62,144,3,0,0,115,2,0,0,0,6,1,122,49, - 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, - 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, - 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, - 62,41,4,114,38,0,0,0,114,35,0,0,0,218,3,97, - 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, - 70,70,73,88,69,83,41,2,114,100,0,0,0,114,119,0, - 0,0,114,4,0,0,0,41,1,114,220,0,0,0,114,5, - 0,0,0,114,153,0,0,0,141,3,0,0,115,6,0,0, - 0,0,2,19,1,18,1,122,30,69,120,116,101,110,115,105, - 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, - 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, - 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,122,63,82,101,116,117,114,110,32, - 78,111,110,101,32,97,115,32,97,110,32,101,120,116,101,110, - 115,105,111,110,32,109,111,100,117,108,101,32,99,97,110,110, - 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, - 32,111,98,106,101,99,116,46,78,114,4,0,0,0,41,2, - 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,181,0,0,0,147,3, - 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, - 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, - 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, - 2,0,0,0,1,0,0,0,67,0,0,0,115,4,0,0, - 0,100,1,0,83,41,2,122,53,82,101,116,117,114,110,32, - 78,111,110,101,32,97,115,32,101,120,116,101,110,115,105,111, - 110,32,109,111,100,117,108,101,115,32,104,97,118,101,32,110, - 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, - 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,151,3,0,0,115,2,0,0,0,0,2,122, - 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, - 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, + 104,95,115,116,97,116,115,160,2,0,0,115,2,0,0,0, + 0,11,122,23,83,111,117,114,99,101,76,111,97,100,101,114, + 46,112,97,116,104,95,115,116,97,116,115,99,4,0,0,0, + 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, + 115,16,0,0,0,124,0,0,106,0,0,124,2,0,124,3, + 0,131,2,0,83,41,1,122,228,79,112,116,105,111,110,97, + 108,32,109,101,116,104,111,100,32,119,104,105,99,104,32,119, + 114,105,116,101,115,32,100,97,116,97,32,40,98,121,116,101, + 115,41,32,116,111,32,97,32,102,105,108,101,32,112,97,116, + 104,32,40,97,32,115,116,114,41,46,10,10,32,32,32,32, + 32,32,32,32,73,109,112,108,101,109,101,110,116,105,110,103, + 32,116,104,105,115,32,109,101,116,104,111,100,32,97,108,108, + 111,119,115,32,102,111,114,32,116,104,101,32,119,114,105,116, + 105,110,103,32,111,102,32,98,121,116,101,99,111,100,101,32, + 102,105,108,101,115,46,10,10,32,32,32,32,32,32,32,32, + 84,104,101,32,115,111,117,114,99,101,32,112,97,116,104,32, + 105,115,32,110,101,101,100,101,100,32,105,110,32,111,114,100, + 101,114,32,116,111,32,99,111,114,114,101,99,116,108,121,32, + 116,114,97,110,115,102,101,114,32,112,101,114,109,105,115,115, + 105,111,110,115,10,32,32,32,32,32,32,32,32,41,1,218, + 8,115,101,116,95,100,97,116,97,41,4,114,100,0,0,0, + 114,90,0,0,0,90,10,99,97,99,104,101,95,112,97,116, + 104,114,53,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,15,95,99,97,99,104,101,95,98,121, + 116,101,99,111,100,101,173,2,0,0,115,2,0,0,0,0, + 8,122,28,83,111,117,114,99,101,76,111,97,100,101,114,46, + 95,99,97,99,104,101,95,98,121,116,101,99,111,100,101,99, + 3,0,0,0,0,0,0,0,3,0,0,0,1,0,0,0, + 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,122, + 150,79,112,116,105,111,110,97,108,32,109,101,116,104,111,100, + 32,119,104,105,99,104,32,119,114,105,116,101,115,32,100,97, + 116,97,32,40,98,121,116,101,115,41,32,116,111,32,97,32, + 102,105,108,101,32,112,97,116,104,32,40,97,32,115,116,114, + 41,46,10,10,32,32,32,32,32,32,32,32,73,109,112,108, + 101,109,101,110,116,105,110,103,32,116,104,105,115,32,109,101, + 116,104,111,100,32,97,108,108,111,119,115,32,102,111,114,32, + 116,104,101,32,119,114,105,116,105,110,103,32,111,102,32,98, + 121,116,101,99,111,100,101,32,102,105,108,101,115,46,10,32, + 32,32,32,32,32,32,32,78,114,4,0,0,0,41,3,114, + 100,0,0,0,114,35,0,0,0,114,53,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,192,0, + 0,0,183,2,0,0,115,0,0,0,0,122,21,83,111,117, + 114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97, + 116,97,99,2,0,0,0,0,0,0,0,5,0,0,0,16, + 0,0,0,67,0,0,0,115,105,0,0,0,124,0,0,106, + 0,0,124,1,0,131,1,0,125,2,0,121,19,0,124,0, + 0,106,1,0,124,2,0,131,1,0,125,3,0,87,110,58, + 0,4,116,2,0,107,10,0,114,94,0,1,125,4,0,1, + 122,26,0,116,3,0,100,1,0,100,2,0,124,1,0,131, + 1,1,124,4,0,130,2,0,87,89,100,3,0,100,3,0, + 125,4,0,126,4,0,88,110,1,0,88,116,4,0,124,3, + 0,131,1,0,83,41,4,122,52,67,111,110,99,114,101,116, + 101,32,105,109,112,108,101,109,101,110,116,97,116,105,111,110, + 32,111,102,32,73,110,115,112,101,99,116,76,111,97,100,101, + 114,46,103,101,116,95,115,111,117,114,99,101,46,122,39,115, + 111,117,114,99,101,32,110,111,116,32,97,118,97,105,108,97, + 98,108,101,32,116,104,114,111,117,103,104,32,103,101,116,95, + 100,97,116,97,40,41,114,98,0,0,0,78,41,5,114,151, + 0,0,0,218,8,103,101,116,95,100,97,116,97,114,40,0, + 0,0,114,99,0,0,0,114,149,0,0,0,41,5,114,100, + 0,0,0,114,119,0,0,0,114,35,0,0,0,114,147,0, + 0,0,218,3,101,120,99,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,10,103,101,116,95,115,111,117,114, + 99,101,190,2,0,0,115,14,0,0,0,0,2,15,1,3, + 1,19,1,18,1,9,1,31,1,122,23,83,111,117,114,99, + 101,76,111,97,100,101,114,46,103,101,116,95,115,111,117,114, + 99,101,218,9,95,111,112,116,105,109,105,122,101,114,29,0, + 0,0,99,3,0,0,0,1,0,0,0,4,0,0,0,9, + 0,0,0,67,0,0,0,115,34,0,0,0,116,0,0,106, + 1,0,116,2,0,124,1,0,124,2,0,100,1,0,100,2, + 0,100,3,0,100,4,0,124,3,0,131,4,2,83,41,5, + 122,130,82,101,116,117,114,110,32,116,104,101,32,99,111,100, + 101,32,111,98,106,101,99,116,32,99,111,109,112,105,108,101, + 100,32,102,114,111,109,32,115,111,117,114,99,101,46,10,10, + 32,32,32,32,32,32,32,32,84,104,101,32,39,100,97,116, + 97,39,32,97,114,103,117,109,101,110,116,32,99,97,110,32, + 98,101,32,97,110,121,32,111,98,106,101,99,116,32,116,121, + 112,101,32,116,104,97,116,32,99,111,109,112,105,108,101,40, + 41,32,115,117,112,112,111,114,116,115,46,10,32,32,32,32, + 32,32,32,32,114,183,0,0,0,218,12,100,111,110,116,95, + 105,110,104,101,114,105,116,84,114,68,0,0,0,41,3,114, + 114,0,0,0,114,182,0,0,0,218,7,99,111,109,112,105, + 108,101,41,4,114,100,0,0,0,114,53,0,0,0,114,35, + 0,0,0,114,197,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,14,115,111,117,114,99,101,95, + 116,111,95,99,111,100,101,200,2,0,0,115,4,0,0,0, + 0,5,21,1,122,27,83,111,117,114,99,101,76,111,97,100, + 101,114,46,115,111,117,114,99,101,95,116,111,95,99,111,100, + 101,99,2,0,0,0,0,0,0,0,10,0,0,0,43,0, + 0,0,67,0,0,0,115,183,1,0,0,124,0,0,106,0, + 0,124,1,0,131,1,0,125,2,0,100,1,0,125,3,0, + 121,16,0,116,1,0,124,2,0,131,1,0,125,4,0,87, + 110,24,0,4,116,2,0,107,10,0,114,63,0,1,1,1, + 100,1,0,125,4,0,89,110,205,0,88,121,19,0,124,0, + 0,106,3,0,124,2,0,131,1,0,125,5,0,87,110,18, + 0,4,116,4,0,107,10,0,114,103,0,1,1,1,89,110, + 165,0,88,116,5,0,124,5,0,100,2,0,25,131,1,0, + 125,3,0,121,19,0,124,0,0,106,6,0,124,4,0,131, + 1,0,125,6,0,87,110,18,0,4,116,7,0,107,10,0, + 114,159,0,1,1,1,89,110,109,0,88,121,34,0,116,8, + 0,124,6,0,100,3,0,124,5,0,100,4,0,124,1,0, + 100,5,0,124,4,0,131,1,3,125,7,0,87,110,24,0, + 4,116,9,0,116,10,0,102,2,0,107,10,0,114,220,0, + 1,1,1,89,110,48,0,88,116,11,0,106,12,0,100,6, + 0,124,4,0,124,2,0,131,3,0,1,116,13,0,124,7, + 0,100,4,0,124,1,0,100,7,0,124,4,0,100,8,0, + 124,2,0,131,1,3,83,124,0,0,106,6,0,124,2,0, + 131,1,0,125,8,0,124,0,0,106,14,0,124,8,0,124, + 2,0,131,2,0,125,9,0,116,11,0,106,12,0,100,9, + 0,124,2,0,131,2,0,1,116,15,0,106,16,0,12,114, + 179,1,124,4,0,100,1,0,107,9,0,114,179,1,124,3, + 0,100,1,0,107,9,0,114,179,1,116,17,0,124,9,0, + 124,3,0,116,18,0,124,8,0,131,1,0,131,3,0,125, + 6,0,121,39,0,124,0,0,106,19,0,124,2,0,124,4, + 0,124,6,0,131,3,0,1,116,11,0,106,12,0,100,10, + 0,124,4,0,131,2,0,1,87,110,18,0,4,116,2,0, + 107,10,0,114,178,1,1,1,1,89,110,1,0,88,124,9, + 0,83,41,11,122,190,67,111,110,99,114,101,116,101,32,105, + 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, + 32,73,110,115,112,101,99,116,76,111,97,100,101,114,46,103, + 101,116,95,99,111,100,101,46,10,10,32,32,32,32,32,32, + 32,32,82,101,97,100,105,110,103,32,111,102,32,98,121,116, + 101,99,111,100,101,32,114,101,113,117,105,114,101,115,32,112, + 97,116,104,95,115,116,97,116,115,32,116,111,32,98,101,32, + 105,109,112,108,101,109,101,110,116,101,100,46,32,84,111,32, + 119,114,105,116,101,10,32,32,32,32,32,32,32,32,98,121, + 116,101,99,111,100,101,44,32,115,101,116,95,100,97,116,97, + 32,109,117,115,116,32,97,108,115,111,32,98,101,32,105,109, + 112,108,101,109,101,110,116,101,100,46,10,10,32,32,32,32, + 32,32,32,32,78,114,126,0,0,0,114,132,0,0,0,114, + 98,0,0,0,114,35,0,0,0,122,13,123,125,32,109,97, + 116,99,104,101,115,32,123,125,114,89,0,0,0,114,90,0, + 0,0,122,19,99,111,100,101,32,111,98,106,101,99,116,32, + 102,114,111,109,32,123,125,122,10,119,114,111,116,101,32,123, + 33,114,125,41,20,114,151,0,0,0,114,79,0,0,0,114, + 66,0,0,0,114,191,0,0,0,114,189,0,0,0,114,14, + 0,0,0,114,194,0,0,0,114,40,0,0,0,114,135,0, + 0,0,114,99,0,0,0,114,130,0,0,0,114,114,0,0, + 0,114,129,0,0,0,114,141,0,0,0,114,200,0,0,0, + 114,7,0,0,0,218,19,100,111,110,116,95,119,114,105,116, + 101,95,98,121,116,101,99,111,100,101,114,144,0,0,0,114, + 31,0,0,0,114,193,0,0,0,41,10,114,100,0,0,0, + 114,119,0,0,0,114,90,0,0,0,114,133,0,0,0,114, + 89,0,0,0,218,2,115,116,114,53,0,0,0,218,10,98, + 121,116,101,115,95,100,97,116,97,114,147,0,0,0,90,11, + 99,111,100,101,95,111,98,106,101,99,116,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,208, + 2,0,0,115,78,0,0,0,0,7,15,1,6,1,3,1, + 16,1,13,1,11,2,3,1,19,1,13,1,5,2,16,1, + 3,1,19,1,13,1,5,2,3,1,9,1,12,1,13,1, + 19,1,5,2,12,1,7,1,15,1,6,1,7,1,15,1, + 18,1,16,1,22,1,12,1,9,1,15,1,3,1,19,1, + 20,1,13,1,5,1,122,21,83,111,117,114,99,101,76,111, + 97,100,101,114,46,103,101,116,95,99,111,100,101,78,114,87, + 0,0,0,41,10,114,105,0,0,0,114,104,0,0,0,114, + 106,0,0,0,114,190,0,0,0,114,191,0,0,0,114,193, + 0,0,0,114,192,0,0,0,114,196,0,0,0,114,200,0, + 0,0,114,181,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,188,0,0,0, + 150,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, + 10,12,7,12,10,18,8,114,188,0,0,0,99,0,0,0, + 0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0, + 0,115,112,0,0,0,101,0,0,90,1,0,100,0,0,90, + 2,0,100,1,0,90,3,0,100,2,0,100,3,0,132,0, + 0,90,4,0,100,4,0,100,5,0,132,0,0,90,5,0, + 100,6,0,100,7,0,132,0,0,90,6,0,101,7,0,135, + 0,0,102,1,0,100,8,0,100,9,0,134,0,0,131,1, + 0,90,8,0,101,7,0,100,10,0,100,11,0,132,0,0, + 131,1,0,90,9,0,100,12,0,100,13,0,132,0,0,90, + 10,0,135,0,0,83,41,14,218,10,70,105,108,101,76,111, + 97,100,101,114,122,103,66,97,115,101,32,102,105,108,101,32, + 108,111,97,100,101,114,32,99,108,97,115,115,32,119,104,105, + 99,104,32,105,109,112,108,101,109,101,110,116,115,32,116,104, + 101,32,108,111,97,100,101,114,32,112,114,111,116,111,99,111, + 108,32,109,101,116,104,111,100,115,32,116,104,97,116,10,32, + 32,32,32,114,101,113,117,105,114,101,32,102,105,108,101,32, + 115,121,115,116,101,109,32,117,115,97,103,101,46,99,3,0, + 0,0,0,0,0,0,3,0,0,0,2,0,0,0,67,0, + 0,0,115,22,0,0,0,124,1,0,124,0,0,95,0,0, + 124,2,0,124,0,0,95,1,0,100,1,0,83,41,2,122, + 75,67,97,99,104,101,32,116,104,101,32,109,111,100,117,108, + 101,32,110,97,109,101,32,97,110,100,32,116,104,101,32,112, + 97,116,104,32,116,111,32,116,104,101,32,102,105,108,101,32, + 102,111,117,110,100,32,98,121,32,116,104,101,10,32,32,32, + 32,32,32,32,32,102,105,110,100,101,114,46,78,41,2,114, + 98,0,0,0,114,35,0,0,0,41,3,114,100,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,9,3, + 0,0,115,4,0,0,0,0,3,9,1,122,19,70,105,108, + 101,76,111,97,100,101,114,46,95,95,105,110,105,116,95,95, + 99,2,0,0,0,0,0,0,0,2,0,0,0,2,0,0, + 0,67,0,0,0,115,34,0,0,0,124,0,0,106,0,0, + 124,1,0,106,0,0,107,2,0,111,33,0,124,0,0,106, + 1,0,124,1,0,106,1,0,107,2,0,83,41,1,78,41, + 2,218,9,95,95,99,108,97,115,115,95,95,114,111,0,0, + 0,41,2,114,100,0,0,0,218,5,111,116,104,101,114,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, + 95,95,101,113,95,95,15,3,0,0,115,4,0,0,0,0, + 1,18,1,122,17,70,105,108,101,76,111,97,100,101,114,46, + 95,95,101,113,95,95,99,1,0,0,0,0,0,0,0,1, + 0,0,0,3,0,0,0,67,0,0,0,115,26,0,0,0, + 116,0,0,124,0,0,106,1,0,131,1,0,116,0,0,124, + 0,0,106,2,0,131,1,0,65,83,41,1,78,41,3,218, + 4,104,97,115,104,114,98,0,0,0,114,35,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,19, + 3,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, + 76,111,97,100,101,114,46,95,95,104,97,115,104,95,95,99, + 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, + 3,0,0,0,115,22,0,0,0,116,0,0,116,1,0,124, + 0,0,131,2,0,106,2,0,124,1,0,131,1,0,83,41, + 1,122,100,76,111,97,100,32,97,32,109,111,100,117,108,101, + 32,102,114,111,109,32,97,32,102,105,108,101,46,10,10,32, + 32,32,32,32,32,32,32,84,104,105,115,32,109,101,116,104, + 111,100,32,105,115,32,100,101,112,114,101,99,97,116,101,100, + 46,32,32,85,115,101,32,101,120,101,99,95,109,111,100,117, + 108,101,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,41,3,218,5,115,117,112,101,114, + 114,204,0,0,0,114,187,0,0,0,41,2,114,100,0,0, + 0,114,119,0,0,0,41,1,114,205,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,187,0,0,0,22,3,0,0, + 115,2,0,0,0,0,10,122,22,70,105,108,101,76,111,97, + 100,101,114,46,108,111,97,100,95,109,111,100,117,108,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, 67,0,0,0,115,7,0,0,0,124,0,0,106,0,0,83, 41,1,122,58,82,101,116,117,114,110,32,116,104,101,32,112, @@ -1622,959 +1345,1237 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,155,3,0,0,115,2,0,0,0,0,3, - 122,32,69,120,116,101,110,115,105,111,110,70,105,108,101,76, - 111,97,100,101,114,46,103,101,116,95,102,105,108,101,110,97, - 109,101,78,41,14,114,105,0,0,0,114,104,0,0,0,114, - 106,0,0,0,114,107,0,0,0,114,179,0,0,0,114,207, - 0,0,0,114,209,0,0,0,114,180,0,0,0,114,185,0, - 0,0,114,153,0,0,0,114,181,0,0,0,114,196,0,0, - 0,114,116,0,0,0,114,151,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 218,0,0,0,108,3,0,0,115,20,0,0,0,12,6,6, - 2,12,4,12,4,12,3,12,8,12,6,12,6,12,4,12, - 4,114,218,0,0,0,99,0,0,0,0,0,0,0,0,0, - 0,0,0,2,0,0,0,64,0,0,0,115,130,0,0,0, - 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, - 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, - 0,100,5,0,132,0,0,90,5,0,100,6,0,100,7,0, - 132,0,0,90,6,0,100,8,0,100,9,0,132,0,0,90, - 7,0,100,10,0,100,11,0,132,0,0,90,8,0,100,12, - 0,100,13,0,132,0,0,90,9,0,100,14,0,100,15,0, - 132,0,0,90,10,0,100,16,0,100,17,0,132,0,0,90, - 11,0,100,18,0,100,19,0,132,0,0,90,12,0,100,20, - 0,83,41,21,218,14,95,78,97,109,101,115,112,97,99,101, - 80,97,116,104,97,38,1,0,0,82,101,112,114,101,115,101, - 110,116,115,32,97,32,110,97,109,101,115,112,97,99,101,32, - 112,97,99,107,97,103,101,39,115,32,112,97,116,104,46,32, - 32,73,116,32,117,115,101,115,32,116,104,101,32,109,111,100, - 117,108,101,32,110,97,109,101,10,32,32,32,32,116,111,32, - 102,105,110,100,32,105,116,115,32,112,97,114,101,110,116,32, - 109,111,100,117,108,101,44,32,97,110,100,32,102,114,111,109, - 32,116,104,101,114,101,32,105,116,32,108,111,111,107,115,32, - 117,112,32,116,104,101,32,112,97,114,101,110,116,39,115,10, - 32,32,32,32,95,95,112,97,116,104,95,95,46,32,32,87, - 104,101,110,32,116,104,105,115,32,99,104,97,110,103,101,115, - 44,32,116,104,101,32,109,111,100,117,108,101,39,115,32,111, - 119,110,32,112,97,116,104,32,105,115,32,114,101,99,111,109, - 112,117,116,101,100,44,10,32,32,32,32,117,115,105,110,103, - 32,112,97,116,104,95,102,105,110,100,101,114,46,32,32,70, - 111,114,32,116,111,112,45,108,101,118,101,108,32,109,111,100, - 117,108,101,115,44,32,116,104,101,32,112,97,114,101,110,116, - 32,109,111,100,117,108,101,39,115,32,112,97,116,104,10,32, - 32,32,32,105,115,32,115,121,115,46,112,97,116,104,46,99, - 4,0,0,0,0,0,0,0,4,0,0,0,2,0,0,0, - 67,0,0,0,115,52,0,0,0,124,1,0,124,0,0,95, - 0,0,124,2,0,124,0,0,95,1,0,116,2,0,124,0, - 0,106,3,0,131,0,0,131,1,0,124,0,0,95,4,0, - 124,3,0,124,0,0,95,5,0,100,0,0,83,41,1,78, - 41,6,218,5,95,110,97,109,101,218,5,95,112,97,116,104, - 114,93,0,0,0,218,16,95,103,101,116,95,112,97,114,101, - 110,116,95,112,97,116,104,218,17,95,108,97,115,116,95,112, - 97,114,101,110,116,95,112,97,116,104,218,12,95,112,97,116, - 104,95,102,105,110,100,101,114,41,4,114,100,0,0,0,114, - 98,0,0,0,114,35,0,0,0,218,11,112,97,116,104,95, - 102,105,110,100,101,114,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,168,3,0,0,115,8, - 0,0,0,0,1,9,1,9,1,21,1,122,23,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,46,95,95,105,110, - 105,116,95,95,99,1,0,0,0,0,0,0,0,4,0,0, - 0,3,0,0,0,67,0,0,0,115,53,0,0,0,124,0, - 0,106,0,0,106,1,0,100,1,0,131,1,0,92,3,0, - 125,1,0,125,2,0,125,3,0,124,2,0,100,2,0,107, - 2,0,114,43,0,100,6,0,83,124,1,0,100,5,0,102, - 2,0,83,41,7,122,62,82,101,116,117,114,110,115,32,97, - 32,116,117,112,108,101,32,111,102,32,40,112,97,114,101,110, - 116,45,109,111,100,117,108,101,45,110,97,109,101,44,32,112, - 97,114,101,110,116,45,112,97,116,104,45,97,116,116,114,45, - 110,97,109,101,41,114,58,0,0,0,114,30,0,0,0,114, - 7,0,0,0,114,35,0,0,0,90,8,95,95,112,97,116, - 104,95,95,41,2,122,3,115,121,115,122,4,112,97,116,104, - 41,2,114,225,0,0,0,114,32,0,0,0,41,4,114,100, - 0,0,0,114,216,0,0,0,218,3,100,111,116,90,2,109, - 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,23,95,102,105,110,100,95,112,97,114,101,110,116,95,112, - 97,116,104,95,110,97,109,101,115,174,3,0,0,115,8,0, - 0,0,0,2,27,1,12,2,4,3,122,38,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,102,105,110,100, - 95,112,97,114,101,110,116,95,112,97,116,104,95,110,97,109, - 101,115,99,1,0,0,0,0,0,0,0,3,0,0,0,3, - 0,0,0,67,0,0,0,115,38,0,0,0,124,0,0,106, - 0,0,131,0,0,92,2,0,125,1,0,125,2,0,116,1, - 0,116,2,0,106,3,0,124,1,0,25,124,2,0,131,2, - 0,83,41,1,78,41,4,114,232,0,0,0,114,110,0,0, - 0,114,7,0,0,0,218,7,109,111,100,117,108,101,115,41, - 3,114,100,0,0,0,90,18,112,97,114,101,110,116,95,109, - 111,100,117,108,101,95,110,97,109,101,90,14,112,97,116,104, - 95,97,116,116,114,95,110,97,109,101,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,227,0,0,0,184,3, - 0,0,115,4,0,0,0,0,1,18,1,122,31,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,46,95,103,101,116, - 95,112,97,114,101,110,116,95,112,97,116,104,99,1,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, - 0,115,118,0,0,0,116,0,0,124,0,0,106,1,0,131, - 0,0,131,1,0,125,1,0,124,1,0,124,0,0,106,2, - 0,107,3,0,114,111,0,124,0,0,106,3,0,124,0,0, - 106,4,0,124,1,0,131,2,0,125,2,0,124,2,0,100, - 0,0,107,9,0,114,102,0,124,2,0,106,5,0,100,0, - 0,107,8,0,114,102,0,124,2,0,106,6,0,114,102,0, - 124,2,0,106,6,0,124,0,0,95,7,0,124,1,0,124, - 0,0,95,2,0,124,0,0,106,7,0,83,41,1,78,41, - 8,114,93,0,0,0,114,227,0,0,0,114,228,0,0,0, - 114,229,0,0,0,114,225,0,0,0,114,120,0,0,0,114, - 150,0,0,0,114,226,0,0,0,41,3,114,100,0,0,0, - 90,11,112,97,114,101,110,116,95,112,97,116,104,114,158,0, + 114,151,0,0,0,34,3,0,0,115,2,0,0,0,0,3, + 122,23,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,102,105,108,101,110,97,109,101,99,2,0,0,0,0,0, + 0,0,3,0,0,0,9,0,0,0,67,0,0,0,115,42, + 0,0,0,116,0,0,106,1,0,124,1,0,100,1,0,131, + 2,0,143,17,0,125,2,0,124,2,0,106,2,0,131,0, + 0,83,87,100,2,0,81,82,88,100,2,0,83,41,3,122, + 39,82,101,116,117,114,110,32,116,104,101,32,100,97,116,97, + 32,102,114,111,109,32,112,97,116,104,32,97,115,32,114,97, + 119,32,98,121,116,101,115,46,218,1,114,78,41,3,114,49, + 0,0,0,114,50,0,0,0,90,4,114,101,97,100,41,3, + 114,100,0,0,0,114,35,0,0,0,114,54,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,194, + 0,0,0,39,3,0,0,115,4,0,0,0,0,2,21,1, + 122,19,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,100,97,116,97,41,11,114,105,0,0,0,114,104,0,0, + 0,114,106,0,0,0,114,107,0,0,0,114,179,0,0,0, + 114,207,0,0,0,114,209,0,0,0,114,116,0,0,0,114, + 187,0,0,0,114,151,0,0,0,114,194,0,0,0,114,4, + 0,0,0,114,4,0,0,0,41,1,114,205,0,0,0,114, + 5,0,0,0,114,204,0,0,0,4,3,0,0,115,14,0, + 0,0,12,3,6,2,12,6,12,4,12,3,24,12,18,5, + 114,204,0,0,0,99,0,0,0,0,0,0,0,0,0,0, + 0,0,4,0,0,0,64,0,0,0,115,64,0,0,0,101, + 0,0,90,1,0,100,0,0,90,2,0,100,1,0,90,3, + 0,100,2,0,100,3,0,132,0,0,90,4,0,100,4,0, + 100,5,0,132,0,0,90,5,0,100,6,0,100,7,0,100, + 8,0,100,9,0,132,0,1,90,6,0,100,10,0,83,41, + 11,218,16,83,111,117,114,99,101,70,105,108,101,76,111,97, + 100,101,114,122,62,67,111,110,99,114,101,116,101,32,105,109, + 112,108,101,109,101,110,116,97,116,105,111,110,32,111,102,32, + 83,111,117,114,99,101,76,111,97,100,101,114,32,117,115,105, + 110,103,32,116,104,101,32,102,105,108,101,32,115,121,115,116, + 101,109,46,99,2,0,0,0,0,0,0,0,3,0,0,0, + 4,0,0,0,67,0,0,0,115,34,0,0,0,116,0,0, + 124,1,0,131,1,0,125,2,0,100,1,0,124,2,0,106, + 1,0,100,2,0,124,2,0,106,2,0,105,2,0,83,41, + 3,122,33,82,101,116,117,114,110,32,116,104,101,32,109,101, + 116,97,100,97,116,97,32,102,111,114,32,116,104,101,32,112, + 97,116,104,46,114,126,0,0,0,114,127,0,0,0,41,3, + 114,39,0,0,0,218,8,115,116,95,109,116,105,109,101,90, + 7,115,116,95,115,105,122,101,41,3,114,100,0,0,0,114, + 35,0,0,0,114,202,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,191,0,0,0,49,3,0, + 0,115,4,0,0,0,0,2,12,1,122,27,83,111,117,114, + 99,101,70,105,108,101,76,111,97,100,101,114,46,112,97,116, + 104,95,115,116,97,116,115,99,4,0,0,0,0,0,0,0, + 5,0,0,0,5,0,0,0,67,0,0,0,115,34,0,0, + 0,116,0,0,124,1,0,131,1,0,125,4,0,124,0,0, + 106,1,0,124,2,0,124,3,0,100,1,0,124,4,0,131, + 2,1,83,41,2,78,218,5,95,109,111,100,101,41,2,114, + 97,0,0,0,114,192,0,0,0,41,5,114,100,0,0,0, + 114,90,0,0,0,114,89,0,0,0,114,53,0,0,0,114, + 42,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,193,0,0,0,54,3,0,0,115,4,0,0, + 0,0,2,12,1,122,32,83,111,117,114,99,101,70,105,108, + 101,76,111,97,100,101,114,46,95,99,97,99,104,101,95,98, + 121,116,101,99,111,100,101,114,214,0,0,0,105,182,1,0, + 0,99,3,0,0,0,1,0,0,0,9,0,0,0,17,0, + 0,0,67,0,0,0,115,62,1,0,0,116,0,0,124,1, + 0,131,1,0,92,2,0,125,4,0,125,5,0,103,0,0, + 125,6,0,120,54,0,124,4,0,114,80,0,116,1,0,124, + 4,0,131,1,0,12,114,80,0,116,0,0,124,4,0,131, + 1,0,92,2,0,125,4,0,125,7,0,124,6,0,106,2, + 0,124,7,0,131,1,0,1,113,27,0,87,120,135,0,116, + 3,0,124,6,0,131,1,0,68,93,121,0,125,7,0,116, + 4,0,124,4,0,124,7,0,131,2,0,125,4,0,121,17, + 0,116,5,0,106,6,0,124,4,0,131,1,0,1,87,113, + 94,0,4,116,7,0,107,10,0,114,155,0,1,1,1,119, + 94,0,89,113,94,0,4,116,8,0,107,10,0,114,214,0, + 1,125,8,0,1,122,28,0,116,9,0,106,10,0,100,1, + 0,124,4,0,124,8,0,131,3,0,1,100,2,0,83,87, + 89,100,2,0,100,2,0,125,8,0,126,8,0,88,113,94, + 0,88,113,94,0,87,121,36,0,116,11,0,124,1,0,124, + 2,0,124,3,0,131,3,0,1,116,9,0,106,10,0,100, + 3,0,124,1,0,131,2,0,1,87,110,56,0,4,116,8, + 0,107,10,0,114,57,1,1,125,8,0,1,122,24,0,116, + 9,0,106,10,0,100,1,0,124,1,0,124,8,0,131,3, + 0,1,87,89,100,2,0,100,2,0,125,8,0,126,8,0, + 88,110,1,0,88,100,2,0,83,41,4,122,27,87,114,105, + 116,101,32,98,121,116,101,115,32,100,97,116,97,32,116,111, + 32,97,32,102,105,108,101,46,122,27,99,111,117,108,100,32, + 110,111,116,32,99,114,101,97,116,101,32,123,33,114,125,58, + 32,123,33,114,125,78,122,12,99,114,101,97,116,101,100,32, + 123,33,114,125,41,12,114,38,0,0,0,114,46,0,0,0, + 114,157,0,0,0,114,33,0,0,0,114,28,0,0,0,114, + 3,0,0,0,90,5,109,107,100,105,114,218,15,70,105,108, + 101,69,120,105,115,116,115,69,114,114,111,114,114,40,0,0, + 0,114,114,0,0,0,114,129,0,0,0,114,55,0,0,0, + 41,9,114,100,0,0,0,114,35,0,0,0,114,53,0,0, + 0,114,214,0,0,0,218,6,112,97,114,101,110,116,114,94, + 0,0,0,114,27,0,0,0,114,23,0,0,0,114,195,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,114,101,99,97,108,99,117,108,97,116,101,188, - 3,0,0,115,16,0,0,0,0,2,18,1,15,1,21,3, - 27,1,9,1,12,1,9,1,122,27,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,46,95,114,101,99,97,108,99, - 117,108,97,116,101,99,1,0,0,0,0,0,0,0,1,0, - 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, - 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, - 1,78,41,2,218,4,105,116,101,114,114,234,0,0,0,41, - 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,105,116,101,114,95,95,201, - 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,105,116,101, - 114,95,95,99,1,0,0,0,0,0,0,0,1,0,0,0, - 2,0,0,0,67,0,0,0,115,16,0,0,0,116,0,0, - 124,0,0,106,1,0,131,0,0,131,1,0,83,41,1,78, - 41,2,114,31,0,0,0,114,234,0,0,0,41,1,114,100, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,7,95,95,108,101,110,95,95,204,3,0,0,115, - 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, - 0,0,0,115,16,0,0,0,100,1,0,106,0,0,124,0, - 0,106,1,0,131,1,0,83,41,2,78,122,20,95,78,97, - 109,101,115,112,97,99,101,80,97,116,104,40,123,33,114,125, - 41,41,2,114,47,0,0,0,114,226,0,0,0,41,1,114, - 100,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,95,114,101,112,114,95,95,207,3,0, - 0,115,2,0,0,0,0,1,122,23,95,78,97,109,101,115, - 112,97,99,101,80,97,116,104,46,95,95,114,101,112,114,95, - 95,99,2,0,0,0,0,0,0,0,2,0,0,0,2,0, - 0,0,67,0,0,0,115,16,0,0,0,124,1,0,124,0, - 0,106,0,0,131,0,0,107,6,0,83,41,1,78,41,1, - 114,234,0,0,0,41,2,114,100,0,0,0,218,4,105,116, - 101,109,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,12,95,95,99,111,110,116,97,105,110,115,95,95,210, - 3,0,0,115,2,0,0,0,0,1,122,27,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,95,95,99,111,110, - 116,97,105,110,115,95,95,99,2,0,0,0,0,0,0,0, - 2,0,0,0,2,0,0,0,67,0,0,0,115,20,0,0, - 0,124,0,0,106,0,0,106,1,0,124,1,0,131,1,0, - 1,100,0,0,83,41,1,78,41,2,114,226,0,0,0,114, - 157,0,0,0,41,2,114,100,0,0,0,114,239,0,0,0, + 0,114,192,0,0,0,59,3,0,0,115,42,0,0,0,0, + 2,18,1,6,2,22,1,18,1,17,2,19,1,15,1,3, + 1,17,1,13,2,7,1,18,3,9,1,10,1,27,1,3, + 1,16,1,20,1,18,2,12,1,122,25,83,111,117,114,99, + 101,70,105,108,101,76,111,97,100,101,114,46,115,101,116,95, + 100,97,116,97,78,41,7,114,105,0,0,0,114,104,0,0, + 0,114,106,0,0,0,114,107,0,0,0,114,191,0,0,0, + 114,193,0,0,0,114,192,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,212, + 0,0,0,45,3,0,0,115,8,0,0,0,12,2,6,2, + 12,5,12,5,114,212,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,46, + 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, + 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, + 0,100,4,0,100,5,0,132,0,0,90,5,0,100,6,0, + 83,41,7,218,20,83,111,117,114,99,101,108,101,115,115,70, + 105,108,101,76,111,97,100,101,114,122,45,76,111,97,100,101, + 114,32,119,104,105,99,104,32,104,97,110,100,108,101,115,32, + 115,111,117,114,99,101,108,101,115,115,32,102,105,108,101,32, + 105,109,112,111,114,116,115,46,99,2,0,0,0,0,0,0, + 0,5,0,0,0,6,0,0,0,67,0,0,0,115,76,0, + 0,0,124,0,0,106,0,0,124,1,0,131,1,0,125,2, + 0,124,0,0,106,1,0,124,2,0,131,1,0,125,3,0, + 116,2,0,124,3,0,100,1,0,124,1,0,100,2,0,124, + 2,0,131,1,2,125,4,0,116,3,0,124,4,0,100,1, + 0,124,1,0,100,3,0,124,2,0,131,1,2,83,41,4, + 78,114,98,0,0,0,114,35,0,0,0,114,89,0,0,0, + 41,4,114,151,0,0,0,114,194,0,0,0,114,135,0,0, + 0,114,141,0,0,0,41,5,114,100,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,53,0,0,0,114,203,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,181,0,0,0,94,3,0,0,115,8,0,0,0,0,1, + 15,1,15,1,24,1,122,29,83,111,117,114,99,101,108,101, + 115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,116, + 95,99,111,100,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,1,0,0,0,67,0,0,0,115,4,0,0,0,100, + 1,0,83,41,2,122,39,82,101,116,117,114,110,32,78,111, + 110,101,32,97,115,32,116,104,101,114,101,32,105,115,32,110, + 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 157,0,0,0,213,3,0,0,115,2,0,0,0,0,1,122, - 21,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, - 97,112,112,101,110,100,78,41,13,114,105,0,0,0,114,104, - 0,0,0,114,106,0,0,0,114,107,0,0,0,114,179,0, - 0,0,114,232,0,0,0,114,227,0,0,0,114,234,0,0, - 0,114,236,0,0,0,114,237,0,0,0,114,238,0,0,0, - 114,240,0,0,0,114,157,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,224, - 0,0,0,161,3,0,0,115,20,0,0,0,12,5,6,2, - 12,6,12,10,12,4,12,13,12,3,12,3,12,3,12,3, - 114,224,0,0,0,99,0,0,0,0,0,0,0,0,0,0, - 0,0,3,0,0,0,64,0,0,0,115,118,0,0,0,101, - 0,0,90,1,0,100,0,0,90,2,0,100,1,0,100,2, - 0,132,0,0,90,3,0,101,4,0,100,3,0,100,4,0, - 132,0,0,131,1,0,90,5,0,100,5,0,100,6,0,132, - 0,0,90,6,0,100,7,0,100,8,0,132,0,0,90,7, - 0,100,9,0,100,10,0,132,0,0,90,8,0,100,11,0, - 100,12,0,132,0,0,90,9,0,100,13,0,100,14,0,132, - 0,0,90,10,0,100,15,0,100,16,0,132,0,0,90,11, - 0,100,17,0,83,41,18,218,16,95,78,97,109,101,115,112, - 97,99,101,76,111,97,100,101,114,99,4,0,0,0,0,0, - 0,0,4,0,0,0,4,0,0,0,67,0,0,0,115,25, - 0,0,0,116,0,0,124,1,0,124,2,0,124,3,0,131, - 3,0,124,0,0,95,1,0,100,0,0,83,41,1,78,41, - 2,114,224,0,0,0,114,226,0,0,0,41,4,114,100,0, - 0,0,114,98,0,0,0,114,35,0,0,0,114,230,0,0, + 196,0,0,0,100,3,0,0,115,2,0,0,0,0,2,122, + 31,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, + 111,97,100,101,114,46,103,101,116,95,115,111,117,114,99,101, + 78,41,6,114,105,0,0,0,114,104,0,0,0,114,106,0, + 0,0,114,107,0,0,0,114,181,0,0,0,114,196,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,217,0,0,0,90,3,0,0,115,6, + 0,0,0,12,2,6,2,12,6,114,217,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, + 0,0,0,115,136,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,90,3,0,100,2,0,100,3,0, + 132,0,0,90,4,0,100,4,0,100,5,0,132,0,0,90, + 5,0,100,6,0,100,7,0,132,0,0,90,6,0,100,8, + 0,100,9,0,132,0,0,90,7,0,100,10,0,100,11,0, + 132,0,0,90,8,0,100,12,0,100,13,0,132,0,0,90, + 9,0,100,14,0,100,15,0,132,0,0,90,10,0,100,16, + 0,100,17,0,132,0,0,90,11,0,101,12,0,100,18,0, + 100,19,0,132,0,0,131,1,0,90,13,0,100,20,0,83, + 41,21,218,19,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,122,93,76,111,97,100,101,114,32, + 102,111,114,32,101,120,116,101,110,115,105,111,110,32,109,111, + 100,117,108,101,115,46,10,10,32,32,32,32,84,104,101,32, + 99,111,110,115,116,114,117,99,116,111,114,32,105,115,32,100, + 101,115,105,103,110,101,100,32,116,111,32,119,111,114,107,32, + 119,105,116,104,32,70,105,108,101,70,105,110,100,101,114,46, + 10,10,32,32,32,32,99,3,0,0,0,0,0,0,0,3, + 0,0,0,2,0,0,0,67,0,0,0,115,22,0,0,0, + 124,1,0,124,0,0,95,0,0,124,2,0,124,0,0,95, + 1,0,100,0,0,83,41,1,78,41,2,114,98,0,0,0, + 114,35,0,0,0,41,3,114,100,0,0,0,114,98,0,0, + 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,179,0,0,0,117,3,0,0,115,4, + 0,0,0,0,1,9,1,122,28,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,105, + 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, + 0,0,2,0,0,0,67,0,0,0,115,34,0,0,0,124, + 0,0,106,0,0,124,1,0,106,0,0,107,2,0,111,33, + 0,124,0,0,106,1,0,124,1,0,106,1,0,107,2,0, + 83,41,1,78,41,2,114,205,0,0,0,114,111,0,0,0, + 41,2,114,100,0,0,0,114,206,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,207,0,0,0, + 121,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, + 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, + 101,114,46,95,95,101,113,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,26, + 0,0,0,116,0,0,124,0,0,106,1,0,131,1,0,116, + 0,0,124,0,0,106,2,0,131,1,0,65,83,41,1,78, + 41,3,114,208,0,0,0,114,98,0,0,0,114,35,0,0, + 0,41,1,114,100,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,209,0,0,0,125,3,0,0, + 115,2,0,0,0,0,1,122,28,69,120,116,101,110,115,105, + 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,104, + 97,115,104,95,95,99,2,0,0,0,0,0,0,0,3,0, + 0,0,4,0,0,0,67,0,0,0,115,50,0,0,0,116, + 0,0,106,1,0,116,2,0,106,3,0,124,1,0,131,2, + 0,125,2,0,116,0,0,106,4,0,100,1,0,124,1,0, + 106,5,0,124,0,0,106,6,0,131,3,0,1,124,2,0, + 83,41,2,122,38,67,114,101,97,116,101,32,97,110,32,117, + 110,105,116,105,97,108,105,122,101,100,32,101,120,116,101,110, + 115,105,111,110,32,109,111,100,117,108,101,122,38,101,120,116, + 101,110,115,105,111,110,32,109,111,100,117,108,101,32,123,33, + 114,125,32,108,111,97,100,101,100,32,102,114,111,109,32,123, + 33,114,125,41,7,114,114,0,0,0,114,182,0,0,0,114, + 139,0,0,0,90,14,99,114,101,97,116,101,95,100,121,110, + 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, + 0,0,0,41,3,114,100,0,0,0,114,158,0,0,0,114, + 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,180,0,0,0,128,3,0,0,115,10,0,0, + 0,0,2,6,1,15,1,9,1,16,1,122,33,69,120,116, + 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, + 46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,2, + 0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,67, + 0,0,0,115,48,0,0,0,116,0,0,106,1,0,116,2, + 0,106,3,0,124,1,0,131,2,0,1,116,0,0,106,4, + 0,100,1,0,124,0,0,106,5,0,124,0,0,106,6,0, + 131,3,0,1,100,2,0,83,41,3,122,30,73,110,105,116, + 105,97,108,105,122,101,32,97,110,32,101,120,116,101,110,115, + 105,111,110,32,109,111,100,117,108,101,122,40,101,120,116,101, + 110,115,105,111,110,32,109,111,100,117,108,101,32,123,33,114, + 125,32,101,120,101,99,117,116,101,100,32,102,114,111,109,32, + 123,33,114,125,78,41,7,114,114,0,0,0,114,182,0,0, + 0,114,139,0,0,0,90,12,101,120,101,99,95,100,121,110, + 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, + 0,0,0,41,2,114,100,0,0,0,114,184,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,185, + 0,0,0,136,3,0,0,115,6,0,0,0,0,2,19,1, + 9,1,122,31,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 4,0,0,0,3,0,0,0,115,48,0,0,0,116,0,0, + 124,0,0,106,1,0,131,1,0,100,1,0,25,137,0,0, + 116,2,0,135,0,0,102,1,0,100,2,0,100,3,0,134, + 0,0,116,3,0,68,131,1,0,131,1,0,83,41,4,122, + 49,82,101,116,117,114,110,32,84,114,117,101,32,105,102,32, + 116,104,101,32,101,120,116,101,110,115,105,111,110,32,109,111, + 100,117,108,101,32,105,115,32,97,32,112,97,99,107,97,103, + 101,46,114,29,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,4,0,0,0,51,0,0,0,115,31,0,0, + 0,124,0,0,93,21,0,125,1,0,136,0,0,100,0,0, + 124,1,0,23,107,2,0,86,1,113,3,0,100,1,0,83, + 41,2,114,179,0,0,0,78,114,4,0,0,0,41,2,114, + 22,0,0,0,218,6,115,117,102,102,105,120,41,1,218,9, + 102,105,108,101,95,110,97,109,101,114,4,0,0,0,114,5, + 0,0,0,250,9,60,103,101,110,101,120,112,114,62,145,3, + 0,0,115,2,0,0,0,6,1,122,49,69,120,116,101,110, + 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,105, + 115,95,112,97,99,107,97,103,101,46,60,108,111,99,97,108, + 115,62,46,60,103,101,110,101,120,112,114,62,41,4,114,38, + 0,0,0,114,35,0,0,0,218,3,97,110,121,218,18,69, + 88,84,69,78,83,73,79,78,95,83,85,70,70,73,88,69, + 83,41,2,114,100,0,0,0,114,119,0,0,0,114,4,0, + 0,0,41,1,114,220,0,0,0,114,5,0,0,0,114,153, + 0,0,0,142,3,0,0,115,6,0,0,0,0,2,19,1, + 18,1,122,30,69,120,116,101,110,115,105,111,110,70,105,108, + 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, + 103,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, + 0,0,0,67,0,0,0,115,4,0,0,0,100,1,0,83, + 41,2,122,63,82,101,116,117,114,110,32,78,111,110,101,32, + 97,115,32,97,110,32,101,120,116,101,110,115,105,111,110,32, + 109,111,100,117,108,101,32,99,97,110,110,111,116,32,99,114, + 101,97,116,101,32,97,32,99,111,100,101,32,111,98,106,101, + 99,116,46,78,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,181,0,0,0,148,3,0,0,115,2,0, + 0,0,0,2,122,28,69,120,116,101,110,115,105,111,110,70, + 105,108,101,76,111,97,100,101,114,46,103,101,116,95,99,111, + 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, + 0,0,0,67,0,0,0,115,4,0,0,0,100,1,0,83, + 41,2,122,53,82,101,116,117,114,110,32,78,111,110,101,32, + 97,115,32,101,120,116,101,110,115,105,111,110,32,109,111,100, + 117,108,101,115,32,104,97,118,101,32,110,111,32,115,111,117, + 114,99,101,32,99,111,100,101,46,78,114,4,0,0,0,41, + 2,114,100,0,0,0,114,119,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,152, + 3,0,0,115,2,0,0,0,0,2,122,30,69,120,116,101, + 110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46, + 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, + 0,0,0,2,0,0,0,1,0,0,0,67,0,0,0,115, + 7,0,0,0,124,0,0,106,0,0,83,41,1,122,58,82, + 101,116,117,114,110,32,116,104,101,32,112,97,116,104,32,116, + 111,32,116,104,101,32,115,111,117,114,99,101,32,102,105,108, + 101,32,97,115,32,102,111,117,110,100,32,98,121,32,116,104, + 101,32,102,105,110,100,101,114,46,41,1,114,35,0,0,0, + 41,2,114,100,0,0,0,114,119,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,151,0,0,0, + 156,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, + 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, + 46,103,101,116,95,102,105,108,101,110,97,109,101,78,41,14, + 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, + 107,0,0,0,114,179,0,0,0,114,207,0,0,0,114,209, + 0,0,0,114,180,0,0,0,114,185,0,0,0,114,153,0, + 0,0,114,181,0,0,0,114,196,0,0,0,114,116,0,0, + 0,114,151,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,109, + 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, + 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, + 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, + 0,0,64,0,0,0,115,130,0,0,0,101,0,0,90,1, + 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, + 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, + 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, + 0,100,8,0,100,9,0,132,0,0,90,7,0,100,10,0, + 100,11,0,132,0,0,90,8,0,100,12,0,100,13,0,132, + 0,0,90,9,0,100,14,0,100,15,0,132,0,0,90,10, + 0,100,16,0,100,17,0,132,0,0,90,11,0,100,18,0, + 100,19,0,132,0,0,90,12,0,100,20,0,83,41,21,218, + 14,95,78,97,109,101,115,112,97,99,101,80,97,116,104,97, + 38,1,0,0,82,101,112,114,101,115,101,110,116,115,32,97, + 32,110,97,109,101,115,112,97,99,101,32,112,97,99,107,97, + 103,101,39,115,32,112,97,116,104,46,32,32,73,116,32,117, + 115,101,115,32,116,104,101,32,109,111,100,117,108,101,32,110, + 97,109,101,10,32,32,32,32,116,111,32,102,105,110,100,32, + 105,116,115,32,112,97,114,101,110,116,32,109,111,100,117,108, + 101,44,32,97,110,100,32,102,114,111,109,32,116,104,101,114, + 101,32,105,116,32,108,111,111,107,115,32,117,112,32,116,104, + 101,32,112,97,114,101,110,116,39,115,10,32,32,32,32,95, + 95,112,97,116,104,95,95,46,32,32,87,104,101,110,32,116, + 104,105,115,32,99,104,97,110,103,101,115,44,32,116,104,101, + 32,109,111,100,117,108,101,39,115,32,111,119,110,32,112,97, + 116,104,32,105,115,32,114,101,99,111,109,112,117,116,101,100, + 44,10,32,32,32,32,117,115,105,110,103,32,112,97,116,104, + 95,102,105,110,100,101,114,46,32,32,70,111,114,32,116,111, + 112,45,108,101,118,101,108,32,109,111,100,117,108,101,115,44, + 32,116,104,101,32,112,97,114,101,110,116,32,109,111,100,117, + 108,101,39,115,32,112,97,116,104,10,32,32,32,32,105,115, + 32,115,121,115,46,112,97,116,104,46,99,4,0,0,0,0, + 0,0,0,4,0,0,0,2,0,0,0,67,0,0,0,115, + 52,0,0,0,124,1,0,124,0,0,95,0,0,124,2,0, + 124,0,0,95,1,0,116,2,0,124,0,0,106,3,0,131, + 0,0,131,1,0,124,0,0,95,4,0,124,3,0,124,0, + 0,95,5,0,100,0,0,83,41,1,78,41,6,218,5,95, + 110,97,109,101,218,5,95,112,97,116,104,114,93,0,0,0, + 218,16,95,103,101,116,95,112,97,114,101,110,116,95,112,97, + 116,104,218,17,95,108,97,115,116,95,112,97,114,101,110,116, + 95,112,97,116,104,218,12,95,112,97,116,104,95,102,105,110, + 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, + 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, + 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,179,0,0,0,169,3,0,0,115,8,0,0,0,0,1, + 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, + 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, + 67,0,0,0,115,53,0,0,0,124,0,0,106,0,0,106, + 1,0,100,1,0,131,1,0,92,3,0,125,1,0,125,2, + 0,125,3,0,124,2,0,100,2,0,107,2,0,114,43,0, + 100,6,0,83,124,1,0,100,5,0,102,2,0,83,41,7, + 122,62,82,101,116,117,114,110,115,32,97,32,116,117,112,108, + 101,32,111,102,32,40,112,97,114,101,110,116,45,109,111,100, + 117,108,101,45,110,97,109,101,44,32,112,97,114,101,110,116, + 45,112,97,116,104,45,97,116,116,114,45,110,97,109,101,41, + 114,58,0,0,0,114,30,0,0,0,114,7,0,0,0,114, + 35,0,0,0,90,8,95,95,112,97,116,104,95,95,41,2, + 122,3,115,121,115,122,4,112,97,116,104,41,2,114,225,0, + 0,0,114,32,0,0,0,41,4,114,100,0,0,0,114,216, + 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, + 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, + 97,109,101,115,175,3,0,0,115,8,0,0,0,0,2,27, + 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, + 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, + 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, + 0,0,115,38,0,0,0,124,0,0,106,0,0,131,0,0, + 92,2,0,125,1,0,125,2,0,116,1,0,116,2,0,106, + 3,0,124,1,0,25,124,2,0,131,2,0,83,41,1,78, + 41,4,114,232,0,0,0,114,110,0,0,0,114,7,0,0, + 0,218,7,109,111,100,117,108,101,115,41,3,114,100,0,0, + 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, + 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, + 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,227,0,0,0,185,3,0,0,115,4,0, + 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, + 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, + 3,0,0,0,3,0,0,0,67,0,0,0,115,118,0,0, + 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, + 125,1,0,124,1,0,124,0,0,106,2,0,107,3,0,114, + 111,0,124,0,0,106,3,0,124,0,0,106,4,0,124,1, + 0,131,2,0,125,2,0,124,2,0,100,0,0,107,9,0, + 114,102,0,124,2,0,106,5,0,100,0,0,107,8,0,114, + 102,0,124,2,0,106,6,0,114,102,0,124,2,0,106,6, + 0,124,0,0,95,7,0,124,1,0,124,0,0,95,2,0, + 124,0,0,106,7,0,83,41,1,78,41,8,114,93,0,0, + 0,114,227,0,0,0,114,228,0,0,0,114,229,0,0,0, + 114,225,0,0,0,114,120,0,0,0,114,150,0,0,0,114, + 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, + 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, + 101,99,97,108,99,117,108,97,116,101,189,3,0,0,115,16, + 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, + 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, + 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, + 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, + 0,67,0,0,0,115,16,0,0,0,116,0,0,124,0,0, + 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, + 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,219,3,0,0,115,2,0,0,0,0,1, + 218,8,95,95,105,116,101,114,95,95,202,3,0,0,115,2, + 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, + 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, + 0,0,0,115,16,0,0,0,116,0,0,124,0,0,106,1, + 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, + 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, + 95,108,101,110,95,95,205,3,0,0,115,2,0,0,0,0, + 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, + 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, + 1,0,83,41,2,78,122,20,95,78,97,109,101,115,112,97, + 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, + 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, + 95,95,114,101,112,114,95,95,208,3,0,0,115,2,0,0, + 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, + 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, + 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, + 0,115,16,0,0,0,124,1,0,124,0,0,106,0,0,131, + 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, + 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, + 99,111,110,116,97,105,110,115,95,95,211,3,0,0,115,2, + 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, + 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, + 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, + 0,0,0,67,0,0,0,115,20,0,0,0,124,0,0,106, + 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, + 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, + 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,214, + 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, + 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, + 0,0,0,114,107,0,0,0,114,179,0,0,0,114,232,0, + 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, + 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, + 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,224,0,0,0,162,3, + 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, + 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, + 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, + 0,64,0,0,0,115,118,0,0,0,101,0,0,90,1,0, + 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, + 3,0,101,4,0,100,3,0,100,4,0,132,0,0,131,1, + 0,90,5,0,100,5,0,100,6,0,132,0,0,90,6,0, + 100,7,0,100,8,0,132,0,0,90,7,0,100,9,0,100, + 10,0,132,0,0,90,8,0,100,11,0,100,12,0,132,0, + 0,90,9,0,100,13,0,100,14,0,132,0,0,90,10,0, + 100,15,0,100,16,0,132,0,0,90,11,0,100,17,0,83, + 41,18,218,16,95,78,97,109,101,115,112,97,99,101,76,111, + 97,100,101,114,99,4,0,0,0,0,0,0,0,4,0,0, + 0,4,0,0,0,67,0,0,0,115,25,0,0,0,116,0, + 0,124,1,0,124,2,0,124,3,0,131,3,0,124,0,0, + 95,1,0,100,0,0,83,41,1,78,41,2,114,224,0,0, + 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, + 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, + 220,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, + 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, + 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, + 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, + 100,1,0,106,0,0,124,1,0,106,1,0,131,1,0,83, + 41,2,122,115,82,101,116,117,114,110,32,114,101,112,114,32, + 102,111,114,32,116,104,101,32,109,111,100,117,108,101,46,10, + 10,32,32,32,32,32,32,32,32,84,104,101,32,109,101,116, + 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, + 100,46,32,32,84,104,101,32,105,109,112,111,114,116,32,109, + 97,99,104,105,110,101,114,121,32,100,111,101,115,32,116,104, + 101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,32, + 32,32,32,32,32,32,32,122,25,60,109,111,100,117,108,101, + 32,123,33,114,125,32,40,110,97,109,101,115,112,97,99,101, + 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, + 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, + 101,95,114,101,112,114,223,3,0,0,115,2,0,0,0,0, + 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, + 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, + 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, + 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, + 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,153,0,0,0,232,3,0,0,115,2,0,0,0,0, + 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, + 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, + 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, + 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, + 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, + 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,196,0,0,0,235,3,0,0,115,2,0, + 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, + 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, + 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, + 0,0,67,0,0,0,115,22,0,0,0,116,0,0,100,1, + 0,100,2,0,100,3,0,100,4,0,100,5,0,131,3,1, + 83,41,6,78,114,30,0,0,0,122,8,60,115,116,114,105, + 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, + 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,181,0,0,0,238,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,95,95,105,110,105,116,95,95,99,2,0,0,0, - 0,0,0,0,2,0,0,0,2,0,0,0,67,0,0,0, - 115,16,0,0,0,100,1,0,106,0,0,124,1,0,106,1, - 0,131,1,0,83,41,2,122,115,82,101,116,117,114,110,32, - 114,101,112,114,32,102,111,114,32,116,104,101,32,109,111,100, - 117,108,101,46,10,10,32,32,32,32,32,32,32,32,84,104, - 101,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,84,104,101,32,105,109,112, - 111,114,116,32,109,97,99,104,105,110,101,114,121,32,100,111, - 101,115,32,116,104,101,32,106,111,98,32,105,116,115,101,108, - 102,46,10,10,32,32,32,32,32,32,32,32,122,25,60,109, - 111,100,117,108,101,32,123,33,114,125,32,40,110,97,109,101, - 115,112,97,99,101,41,62,41,2,114,47,0,0,0,114,105, - 0,0,0,41,2,114,164,0,0,0,114,184,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 109,111,100,117,108,101,95,114,101,112,114,222,3,0,0,115, - 2,0,0,0,0,7,122,28,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,109,111,100,117,108,101,95, - 114,101,112,114,99,2,0,0,0,0,0,0,0,2,0,0, - 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,1, - 0,83,41,2,78,84,114,4,0,0,0,41,2,114,100,0, - 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,153,0,0,0,231,3,0,0,115, - 2,0,0,0,0,1,122,27,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,105,115,95,112,97,99,107, - 97,103,101,99,2,0,0,0,0,0,0,0,2,0,0,0, - 1,0,0,0,67,0,0,0,115,4,0,0,0,100,1,0, - 83,41,2,78,114,30,0,0,0,114,4,0,0,0,41,2, - 114,100,0,0,0,114,119,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,196,0,0,0,234,3, - 0,0,115,2,0,0,0,0,1,122,27,95,78,97,109,101, - 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, - 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, - 0,0,0,6,0,0,0,67,0,0,0,115,22,0,0,0, - 116,0,0,100,1,0,100,2,0,100,3,0,100,4,0,100, - 5,0,131,3,1,83,41,6,78,114,30,0,0,0,122,8, - 60,115,116,114,105,110,103,62,114,183,0,0,0,114,198,0, - 0,0,84,41,1,114,199,0,0,0,41,2,114,100,0,0, - 0,114,119,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,181,0,0,0,237,3,0,0,115,2, - 0,0,0,0,1,122,25,95,78,97,109,101,115,112,97,99, - 101,76,111,97,100,101,114,46,103,101,116,95,99,111,100,101, - 99,2,0,0,0,0,0,0,0,2,0,0,0,1,0,0, - 0,67,0,0,0,115,4,0,0,0,100,1,0,83,41,2, - 122,42,85,115,101,32,100,101,102,97,117,108,116,32,115,101, - 109,97,110,116,105,99,115,32,102,111,114,32,109,111,100,117, - 108,101,32,99,114,101,97,116,105,111,110,46,78,114,4,0, - 0,0,41,2,114,100,0,0,0,114,158,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,180,0, - 0,0,240,3,0,0,115,0,0,0,0,122,30,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,46,99,114, - 101,97,116,101,95,109,111,100,117,108,101,99,2,0,0,0, + 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,0,0,83,41,1,78,114,4,0,0, - 0,41,2,114,100,0,0,0,114,184,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,185,0,0, - 0,243,3,0,0,115,2,0,0,0,0,1,122,28,95,78, - 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,101, - 120,101,99,95,109,111,100,117,108,101,99,2,0,0,0,0, - 0,0,0,2,0,0,0,3,0,0,0,67,0,0,0,115, - 35,0,0,0,116,0,0,106,1,0,100,1,0,124,0,0, - 106,2,0,131,2,0,1,116,0,0,106,3,0,124,0,0, - 124,1,0,131,2,0,83,41,2,122,98,76,111,97,100,32, - 97,32,110,97,109,101,115,112,97,99,101,32,109,111,100,117, - 108,101,46,10,10,32,32,32,32,32,32,32,32,84,104,105, - 115,32,109,101,116,104,111,100,32,105,115,32,100,101,112,114, - 101,99,97,116,101,100,46,32,32,85,115,101,32,101,120,101, - 99,95,109,111,100,117,108,101,40,41,32,105,110,115,116,101, - 97,100,46,10,10,32,32,32,32,32,32,32,32,122,38,110, - 97,109,101,115,112,97,99,101,32,109,111,100,117,108,101,32, - 108,111,97,100,101,100,32,119,105,116,104,32,112,97,116,104, - 32,123,33,114,125,41,4,114,114,0,0,0,114,129,0,0, - 0,114,226,0,0,0,114,186,0,0,0,41,2,114,100,0, - 0,0,114,119,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,187,0,0,0,246,3,0,0,115, - 6,0,0,0,0,7,9,1,10,1,122,28,95,78,97,109, - 101,115,112,97,99,101,76,111,97,100,101,114,46,108,111,97, - 100,95,109,111,100,117,108,101,78,41,12,114,105,0,0,0, - 114,104,0,0,0,114,106,0,0,0,114,179,0,0,0,114, - 177,0,0,0,114,242,0,0,0,114,153,0,0,0,114,196, - 0,0,0,114,181,0,0,0,114,180,0,0,0,114,185,0, - 0,0,114,187,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,241,0,0,0, - 218,3,0,0,115,16,0,0,0,12,1,12,3,18,9,12, - 3,12,3,12,3,12,3,12,3,114,241,0,0,0,99,0, - 0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,64, - 0,0,0,115,160,0,0,0,101,0,0,90,1,0,100,0, - 0,90,2,0,100,1,0,90,3,0,101,4,0,100,2,0, - 100,3,0,132,0,0,131,1,0,90,5,0,101,4,0,100, - 4,0,100,5,0,132,0,0,131,1,0,90,6,0,101,4, - 0,100,6,0,100,7,0,132,0,0,131,1,0,90,7,0, - 101,4,0,100,8,0,100,9,0,132,0,0,131,1,0,90, - 8,0,101,4,0,100,10,0,100,11,0,100,12,0,132,1, - 0,131,1,0,90,9,0,101,4,0,100,10,0,100,10,0, - 100,13,0,100,14,0,132,2,0,131,1,0,90,10,0,101, - 4,0,100,10,0,100,15,0,100,16,0,132,1,0,131,1, - 0,90,11,0,100,10,0,83,41,17,218,10,80,97,116,104, - 70,105,110,100,101,114,122,62,77,101,116,97,32,112,97,116, - 104,32,102,105,110,100,101,114,32,102,111,114,32,115,121,115, - 46,112,97,116,104,32,97,110,100,32,112,97,99,107,97,103, - 101,32,95,95,112,97,116,104,95,95,32,97,116,116,114,105, - 98,117,116,101,115,46,99,1,0,0,0,0,0,0,0,2, - 0,0,0,4,0,0,0,67,0,0,0,115,55,0,0,0, - 120,48,0,116,0,0,106,1,0,106,2,0,131,0,0,68, - 93,31,0,125,1,0,116,3,0,124,1,0,100,1,0,131, - 2,0,114,16,0,124,1,0,106,4,0,131,0,0,1,113, - 16,0,87,100,2,0,83,41,3,122,125,67,97,108,108,32, - 116,104,101,32,105,110,118,97,108,105,100,97,116,101,95,99, - 97,99,104,101,115,40,41,32,109,101,116,104,111,100,32,111, - 110,32,97,108,108,32,112,97,116,104,32,101,110,116,114,121, - 32,102,105,110,100,101,114,115,10,32,32,32,32,32,32,32, - 32,115,116,111,114,101,100,32,105,110,32,115,121,115,46,112, - 97,116,104,95,105,109,112,111,114,116,101,114,95,99,97,99, - 104,101,115,32,40,119,104,101,114,101,32,105,109,112,108,101, - 109,101,110,116,101,100,41,46,218,17,105,110,118,97,108,105, - 100,97,116,101,95,99,97,99,104,101,115,78,41,5,114,7, - 0,0,0,218,19,112,97,116,104,95,105,109,112,111,114,116, - 101,114,95,99,97,99,104,101,218,6,118,97,108,117,101,115, - 114,108,0,0,0,114,244,0,0,0,41,2,114,164,0,0, - 0,218,6,102,105,110,100,101,114,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,244,0,0,0,8,4,0, - 0,115,6,0,0,0,0,4,22,1,15,1,122,28,80,97, - 116,104,70,105,110,100,101,114,46,105,110,118,97,108,105,100, - 97,116,101,95,99,97,99,104,101,115,99,2,0,0,0,0, - 0,0,0,3,0,0,0,12,0,0,0,67,0,0,0,115, - 107,0,0,0,116,0,0,106,1,0,100,1,0,107,9,0, - 114,41,0,116,0,0,106,1,0,12,114,41,0,116,2,0, - 106,3,0,100,2,0,116,4,0,131,2,0,1,120,59,0, - 116,0,0,106,1,0,68,93,44,0,125,2,0,121,14,0, - 124,2,0,124,1,0,131,1,0,83,87,113,51,0,4,116, - 5,0,107,10,0,114,94,0,1,1,1,119,51,0,89,113, - 51,0,88,113,51,0,87,100,1,0,83,100,1,0,83,41, - 3,122,113,83,101,97,114,99,104,32,115,101,113,117,101,110, - 99,101,32,111,102,32,104,111,111,107,115,32,102,111,114,32, - 97,32,102,105,110,100,101,114,32,102,111,114,32,39,112,97, - 116,104,39,46,10,10,32,32,32,32,32,32,32,32,73,102, - 32,39,104,111,111,107,115,39,32,105,115,32,102,97,108,115, - 101,32,116,104,101,110,32,117,115,101,32,115,121,115,46,112, - 97,116,104,95,104,111,111,107,115,46,10,10,32,32,32,32, - 32,32,32,32,78,122,23,115,121,115,46,112,97,116,104,95, - 104,111,111,107,115,32,105,115,32,101,109,112,116,121,41,6, - 114,7,0,0,0,218,10,112,97,116,104,95,104,111,111,107, - 115,114,60,0,0,0,114,61,0,0,0,114,118,0,0,0, - 114,99,0,0,0,41,3,114,164,0,0,0,114,35,0,0, - 0,90,4,104,111,111,107,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,11,95,112,97,116,104,95,104,111, - 111,107,115,16,4,0,0,115,16,0,0,0,0,7,25,1, - 16,1,16,1,3,1,14,1,13,1,12,2,122,22,80,97, - 116,104,70,105,110,100,101,114,46,95,112,97,116,104,95,104, - 111,111,107,115,99,2,0,0,0,0,0,0,0,3,0,0, - 0,19,0,0,0,67,0,0,0,115,123,0,0,0,124,1, - 0,100,1,0,107,2,0,114,53,0,121,16,0,116,0,0, - 106,1,0,131,0,0,125,1,0,87,110,22,0,4,116,2, - 0,107,10,0,114,52,0,1,1,1,100,2,0,83,89,110, - 1,0,88,121,17,0,116,3,0,106,4,0,124,1,0,25, - 125,2,0,87,110,46,0,4,116,5,0,107,10,0,114,118, - 0,1,1,1,124,0,0,106,6,0,124,1,0,131,1,0, - 125,2,0,124,2,0,116,3,0,106,4,0,124,1,0,60, - 89,110,1,0,88,124,2,0,83,41,3,122,210,71,101,116, - 32,116,104,101,32,102,105,110,100,101,114,32,102,111,114,32, - 116,104,101,32,112,97,116,104,32,101,110,116,114,121,32,102, - 114,111,109,32,115,121,115,46,112,97,116,104,95,105,109,112, - 111,114,116,101,114,95,99,97,99,104,101,46,10,10,32,32, - 32,32,32,32,32,32,73,102,32,116,104,101,32,112,97,116, - 104,32,101,110,116,114,121,32,105,115,32,110,111,116,32,105, - 110,32,116,104,101,32,99,97,99,104,101,44,32,102,105,110, - 100,32,116,104,101,32,97,112,112,114,111,112,114,105,97,116, - 101,32,102,105,110,100,101,114,10,32,32,32,32,32,32,32, - 32,97,110,100,32,99,97,99,104,101,32,105,116,46,32,73, - 102,32,110,111,32,102,105,110,100,101,114,32,105,115,32,97, - 118,97,105,108,97,98,108,101,44,32,115,116,111,114,101,32, - 78,111,110,101,46,10,10,32,32,32,32,32,32,32,32,114, - 30,0,0,0,78,41,7,114,3,0,0,0,114,45,0,0, - 0,218,17,70,105,108,101,78,111,116,70,111,117,110,100,69, - 114,114,111,114,114,7,0,0,0,114,245,0,0,0,114,131, - 0,0,0,114,249,0,0,0,41,3,114,164,0,0,0,114, - 35,0,0,0,114,247,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,20,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,33,4, - 0,0,115,22,0,0,0,0,8,12,1,3,1,16,1,13, - 3,9,1,3,1,17,1,13,1,15,1,18,1,122,31,80, - 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,99,3, - 0,0,0,0,0,0,0,6,0,0,0,3,0,0,0,67, - 0,0,0,115,119,0,0,0,116,0,0,124,2,0,100,1, - 0,131,2,0,114,39,0,124,2,0,106,1,0,124,1,0, - 131,1,0,92,2,0,125,3,0,125,4,0,110,21,0,124, - 2,0,106,2,0,124,1,0,131,1,0,125,3,0,103,0, - 0,125,4,0,124,3,0,100,0,0,107,9,0,114,88,0, - 116,3,0,106,4,0,124,1,0,124,3,0,131,2,0,83, - 116,3,0,106,5,0,124,1,0,100,0,0,131,2,0,125, - 5,0,124,4,0,124,5,0,95,6,0,124,5,0,83,41, - 2,78,114,117,0,0,0,41,7,114,108,0,0,0,114,117, - 0,0,0,114,176,0,0,0,114,114,0,0,0,114,173,0, - 0,0,114,154,0,0,0,114,150,0,0,0,41,6,114,164, - 0,0,0,114,119,0,0,0,114,247,0,0,0,114,120,0, - 0,0,114,121,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,16,95,108,101, - 103,97,99,121,95,103,101,116,95,115,112,101,99,55,4,0, - 0,115,18,0,0,0,0,4,15,1,24,2,15,1,6,1, - 12,1,16,1,18,1,9,1,122,27,80,97,116,104,70,105, - 110,100,101,114,46,95,108,101,103,97,99,121,95,103,101,116, - 95,115,112,101,99,78,99,4,0,0,0,0,0,0,0,9, - 0,0,0,5,0,0,0,67,0,0,0,115,243,0,0,0, - 103,0,0,125,4,0,120,230,0,124,2,0,68,93,191,0, - 125,5,0,116,0,0,124,5,0,116,1,0,116,2,0,102, - 2,0,131,2,0,115,43,0,113,13,0,124,0,0,106,3, - 0,124,5,0,131,1,0,125,6,0,124,6,0,100,1,0, - 107,9,0,114,13,0,116,4,0,124,6,0,100,2,0,131, - 2,0,114,106,0,124,6,0,106,5,0,124,1,0,124,3, - 0,131,2,0,125,7,0,110,18,0,124,0,0,106,6,0, - 124,1,0,124,6,0,131,2,0,125,7,0,124,7,0,100, - 1,0,107,8,0,114,139,0,113,13,0,124,7,0,106,7, - 0,100,1,0,107,9,0,114,158,0,124,7,0,83,124,7, - 0,106,8,0,125,8,0,124,8,0,100,1,0,107,8,0, - 114,191,0,116,9,0,100,3,0,131,1,0,130,1,0,124, - 4,0,106,10,0,124,8,0,131,1,0,1,113,13,0,87, - 116,11,0,106,12,0,124,1,0,100,1,0,131,2,0,125, - 7,0,124,4,0,124,7,0,95,8,0,124,7,0,83,100, - 1,0,83,41,4,122,63,70,105,110,100,32,116,104,101,32, - 108,111,97,100,101,114,32,111,114,32,110,97,109,101,115,112, - 97,99,101,95,112,97,116,104,32,102,111,114,32,116,104,105, - 115,32,109,111,100,117,108,101,47,112,97,99,107,97,103,101, - 32,110,97,109,101,46,78,114,175,0,0,0,122,19,115,112, - 101,99,32,109,105,115,115,105,110,103,32,108,111,97,100,101, - 114,41,13,114,137,0,0,0,114,69,0,0,0,218,5,98, - 121,116,101,115,114,251,0,0,0,114,108,0,0,0,114,175, - 0,0,0,114,252,0,0,0,114,120,0,0,0,114,150,0, - 0,0,114,99,0,0,0,114,143,0,0,0,114,114,0,0, - 0,114,154,0,0,0,41,9,114,164,0,0,0,114,119,0, - 0,0,114,35,0,0,0,114,174,0,0,0,218,14,110,97, - 109,101,115,112,97,99,101,95,112,97,116,104,90,5,101,110, - 116,114,121,114,247,0,0,0,114,158,0,0,0,114,121,0, + 115,4,0,0,0,100,1,0,83,41,2,122,42,85,115,101, + 32,100,101,102,97,117,108,116,32,115,101,109,97,110,116,105, + 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, + 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, + 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,180,0,0,0,241,3,0, + 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, + 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, + 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, + 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, + 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, + 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,114,185,0,0,0,244,3,0,0, + 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, + 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, + 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, + 0,0,3,0,0,0,67,0,0,0,115,35,0,0,0,116, + 0,0,106,1,0,100,1,0,124,0,0,106,2,0,131,2, + 0,1,116,0,0,106,3,0,124,0,0,124,1,0,131,2, + 0,83,41,2,122,98,76,111,97,100,32,97,32,110,97,109, + 101,115,112,97,99,101,32,109,111,100,117,108,101,46,10,10, + 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, + 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, + 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, + 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, + 32,32,32,32,32,32,32,32,122,38,110,97,109,101,115,112, + 97,99,101,32,109,111,100,117,108,101,32,108,111,97,100,101, + 100,32,119,105,116,104,32,112,97,116,104,32,123,33,114,125, + 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, + 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,9,95,103,101,116,95,115,112,101,99,70,4,0,0, - 115,40,0,0,0,0,5,6,1,13,1,21,1,3,1,15, - 1,12,1,15,1,21,2,18,1,12,1,3,1,15,1,4, - 1,9,1,12,1,12,5,17,2,18,1,9,1,122,20,80, - 97,116,104,70,105,110,100,101,114,46,95,103,101,116,95,115, - 112,101,99,99,4,0,0,0,0,0,0,0,6,0,0,0, - 4,0,0,0,67,0,0,0,115,140,0,0,0,124,2,0, - 100,1,0,107,8,0,114,21,0,116,0,0,106,1,0,125, - 2,0,124,0,0,106,2,0,124,1,0,124,2,0,124,3, - 0,131,3,0,125,4,0,124,4,0,100,1,0,107,8,0, - 114,58,0,100,1,0,83,124,4,0,106,3,0,100,1,0, - 107,8,0,114,132,0,124,4,0,106,4,0,125,5,0,124, - 5,0,114,125,0,100,2,0,124,4,0,95,5,0,116,6, - 0,124,1,0,124,5,0,124,0,0,106,2,0,131,3,0, - 124,4,0,95,4,0,124,4,0,83,100,1,0,83,110,4, - 0,124,4,0,83,100,1,0,83,41,3,122,98,102,105,110, - 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, - 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, - 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, - 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, - 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,78, - 90,9,110,97,109,101,115,112,97,99,101,41,7,114,7,0, - 0,0,114,35,0,0,0,114,255,0,0,0,114,120,0,0, - 0,114,150,0,0,0,114,152,0,0,0,114,224,0,0,0, - 41,6,114,164,0,0,0,114,119,0,0,0,114,35,0,0, - 0,114,174,0,0,0,114,158,0,0,0,114,254,0,0,0, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 175,0,0,0,102,4,0,0,115,26,0,0,0,0,4,12, - 1,9,1,21,1,12,1,4,1,15,1,9,1,6,3,9, - 1,24,1,4,2,7,2,122,20,80,97,116,104,70,105,110, - 100,101,114,46,102,105,110,100,95,115,112,101,99,99,3,0, - 0,0,0,0,0,0,4,0,0,0,3,0,0,0,67,0, - 0,0,115,41,0,0,0,124,0,0,106,0,0,124,1,0, - 124,2,0,131,2,0,125,3,0,124,3,0,100,1,0,107, - 8,0,114,34,0,100,1,0,83,124,3,0,106,1,0,83, - 41,2,122,170,102,105,110,100,32,116,104,101,32,109,111,100, - 117,108,101,32,111,110,32,115,121,115,46,112,97,116,104,32, - 111,114,32,39,112,97,116,104,39,32,98,97,115,101,100,32, - 111,110,32,115,121,115,46,112,97,116,104,95,104,111,111,107, - 115,32,97,110,100,10,32,32,32,32,32,32,32,32,115,121, - 115,46,112,97,116,104,95,105,109,112,111,114,116,101,114,95, - 99,97,99,104,101,46,10,10,32,32,32,32,32,32,32,32, - 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, - 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, - 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, - 2,114,175,0,0,0,114,120,0,0,0,41,4,114,164,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,176,0,0,0,124,4,0,0,115,8,0,0,0,0,8, - 18,1,12,1,4,1,122,22,80,97,116,104,70,105,110,100, - 101,114,46,102,105,110,100,95,109,111,100,117,108,101,41,12, - 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, - 107,0,0,0,114,177,0,0,0,114,244,0,0,0,114,249, - 0,0,0,114,251,0,0,0,114,252,0,0,0,114,255,0, - 0,0,114,175,0,0,0,114,176,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,243,0,0,0,4,4,0,0,115,22,0,0,0,12,2, - 6,2,18,8,18,17,18,22,18,15,3,1,18,31,3,1, - 21,21,3,1,114,243,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,3,0,0,0,64,0,0,0,115,133, + 0,114,187,0,0,0,247,3,0,0,115,6,0,0,0,0, + 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, + 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, + 114,106,0,0,0,114,179,0,0,0,114,177,0,0,0,114, + 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, + 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,241,0,0,0,219,3,0,0,115, + 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, + 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, + 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,90,3,0,100,2,0,100,3,0,132,0,0,90,4, - 0,100,4,0,100,5,0,132,0,0,90,5,0,101,6,0, - 90,7,0,100,6,0,100,7,0,132,0,0,90,8,0,100, - 8,0,100,9,0,132,0,0,90,9,0,100,10,0,100,11, - 0,100,12,0,132,1,0,90,10,0,100,13,0,100,14,0, - 132,0,0,90,11,0,101,12,0,100,15,0,100,16,0,132, - 0,0,131,1,0,90,13,0,100,17,0,100,18,0,132,0, - 0,90,14,0,100,10,0,83,41,19,218,10,70,105,108,101, - 70,105,110,100,101,114,122,172,70,105,108,101,45,98,97,115, - 101,100,32,102,105,110,100,101,114,46,10,10,32,32,32,32, - 73,110,116,101,114,97,99,116,105,111,110,115,32,119,105,116, - 104,32,116,104,101,32,102,105,108,101,32,115,121,115,116,101, - 109,32,97,114,101,32,99,97,99,104,101,100,32,102,111,114, - 32,112,101,114,102,111,114,109,97,110,99,101,44,32,98,101, - 105,110,103,10,32,32,32,32,114,101,102,114,101,115,104,101, - 100,32,119,104,101,110,32,116,104,101,32,100,105,114,101,99, - 116,111,114,121,32,116,104,101,32,102,105,110,100,101,114,32, - 105,115,32,104,97,110,100,108,105,110,103,32,104,97,115,32, - 98,101,101,110,32,109,111,100,105,102,105,101,100,46,10,10, - 32,32,32,32,99,2,0,0,0,0,0,0,0,5,0,0, - 0,5,0,0,0,7,0,0,0,115,122,0,0,0,103,0, - 0,125,3,0,120,52,0,124,2,0,68,93,44,0,92,2, - 0,137,0,0,125,4,0,124,3,0,106,0,0,135,0,0, - 102,1,0,100,1,0,100,2,0,134,0,0,124,4,0,68, - 131,1,0,131,1,0,1,113,13,0,87,124,3,0,124,0, - 0,95,1,0,124,1,0,112,79,0,100,3,0,124,0,0, - 95,2,0,100,6,0,124,0,0,95,3,0,116,4,0,131, - 0,0,124,0,0,95,5,0,116,4,0,131,0,0,124,0, - 0,95,6,0,100,5,0,83,41,7,122,154,73,110,105,116, - 105,97,108,105,122,101,32,119,105,116,104,32,116,104,101,32, - 112,97,116,104,32,116,111,32,115,101,97,114,99,104,32,111, - 110,32,97,110,100,32,97,32,118,97,114,105,97,98,108,101, - 32,110,117,109,98,101,114,32,111,102,10,32,32,32,32,32, - 32,32,32,50,45,116,117,112,108,101,115,32,99,111,110,116, - 97,105,110,105,110,103,32,116,104,101,32,108,111,97,100,101, - 114,32,97,110,100,32,116,104,101,32,102,105,108,101,32,115, - 117,102,102,105,120,101,115,32,116,104,101,32,108,111,97,100, - 101,114,10,32,32,32,32,32,32,32,32,114,101,99,111,103, - 110,105,122,101,115,46,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,51,0,0,0,115,27,0,0,0, - 124,0,0,93,17,0,125,1,0,124,1,0,136,0,0,102, - 2,0,86,1,113,3,0,100,0,0,83,41,1,78,114,4, - 0,0,0,41,2,114,22,0,0,0,114,219,0,0,0,41, - 1,114,120,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,153,4,0,0,115,2,0,0,0,6,0, - 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, - 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, - 103,101,110,101,120,112,114,62,114,58,0,0,0,114,29,0, - 0,0,78,114,87,0,0,0,41,7,114,143,0,0,0,218, - 8,95,108,111,97,100,101,114,115,114,35,0,0,0,218,11, - 95,112,97,116,104,95,109,116,105,109,101,218,3,115,101,116, - 218,11,95,112,97,116,104,95,99,97,99,104,101,218,19,95, - 114,101,108,97,120,101,100,95,112,97,116,104,95,99,97,99, - 104,101,41,5,114,100,0,0,0,114,35,0,0,0,218,14, - 108,111,97,100,101,114,95,100,101,116,97,105,108,115,90,7, - 108,111,97,100,101,114,115,114,160,0,0,0,114,4,0,0, - 0,41,1,114,120,0,0,0,114,5,0,0,0,114,179,0, - 0,0,147,4,0,0,115,16,0,0,0,0,4,6,1,19, - 1,36,1,9,2,15,1,9,1,12,1,122,19,70,105,108, + 1,0,90,3,0,101,4,0,100,2,0,100,3,0,132,0, + 0,131,1,0,90,5,0,101,4,0,100,4,0,100,5,0, + 132,0,0,131,1,0,90,6,0,101,4,0,100,6,0,100, + 7,0,132,0,0,131,1,0,90,7,0,101,4,0,100,8, + 0,100,9,0,132,0,0,131,1,0,90,8,0,101,4,0, + 100,10,0,100,11,0,100,12,0,132,1,0,131,1,0,90, + 9,0,101,4,0,100,10,0,100,10,0,100,13,0,100,14, + 0,132,2,0,131,1,0,90,10,0,101,4,0,100,10,0, + 100,15,0,100,16,0,132,1,0,131,1,0,90,11,0,100, + 10,0,83,41,17,218,10,80,97,116,104,70,105,110,100,101, + 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, + 100,101,114,32,102,111,114,32,115,121,115,46,112,97,116,104, + 32,97,110,100,32,112,97,99,107,97,103,101,32,95,95,112, + 97,116,104,95,95,32,97,116,116,114,105,98,117,116,101,115, + 46,99,1,0,0,0,0,0,0,0,2,0,0,0,4,0, + 0,0,67,0,0,0,115,55,0,0,0,120,48,0,116,0, + 0,106,1,0,106,2,0,131,0,0,68,93,31,0,125,1, + 0,116,3,0,124,1,0,100,1,0,131,2,0,114,16,0, + 124,1,0,106,4,0,131,0,0,1,113,16,0,87,100,2, + 0,83,41,3,122,125,67,97,108,108,32,116,104,101,32,105, + 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, + 40,41,32,109,101,116,104,111,100,32,111,110,32,97,108,108, + 32,112,97,116,104,32,101,110,116,114,121,32,102,105,110,100, + 101,114,115,10,32,32,32,32,32,32,32,32,115,116,111,114, + 101,100,32,105,110,32,115,121,115,46,112,97,116,104,95,105, + 109,112,111,114,116,101,114,95,99,97,99,104,101,115,32,40, + 119,104,101,114,101,32,105,109,112,108,101,109,101,110,116,101, + 100,41,46,218,17,105,110,118,97,108,105,100,97,116,101,95, + 99,97,99,104,101,115,78,41,5,114,7,0,0,0,218,19, + 112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,97, + 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, + 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, + 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,114,244,0,0,0,9,4,0,0,115,6,0,0, + 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, + 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, + 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, + 0,0,12,0,0,0,67,0,0,0,115,107,0,0,0,116, + 0,0,106,1,0,100,1,0,107,9,0,114,41,0,116,0, + 0,106,1,0,12,114,41,0,116,2,0,106,3,0,100,2, + 0,116,4,0,131,2,0,1,120,59,0,116,0,0,106,1, + 0,68,93,44,0,125,2,0,121,14,0,124,2,0,124,1, + 0,131,1,0,83,87,113,51,0,4,116,5,0,107,10,0, + 114,94,0,1,1,1,119,51,0,89,113,51,0,88,113,51, + 0,87,100,1,0,83,100,1,0,83,41,3,122,113,83,101, + 97,114,99,104,32,115,101,113,117,101,110,99,101,32,111,102, + 32,104,111,111,107,115,32,102,111,114,32,97,32,102,105,110, + 100,101,114,32,102,111,114,32,39,112,97,116,104,39,46,10, + 10,32,32,32,32,32,32,32,32,73,102,32,39,104,111,111, + 107,115,39,32,105,115,32,102,97,108,115,101,32,116,104,101, + 110,32,117,115,101,32,115,121,115,46,112,97,116,104,95,104, + 111,111,107,115,46,10,10,32,32,32,32,32,32,32,32,78, + 122,23,115,121,115,46,112,97,116,104,95,104,111,111,107,115, + 32,105,115,32,101,109,112,116,121,41,6,114,7,0,0,0, + 218,10,112,97,116,104,95,104,111,111,107,115,114,60,0,0, + 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, + 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, + 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,11,95,112,97,116,104,95,104,111,111,107,115,17,4, + 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, + 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, + 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, + 2,0,0,0,0,0,0,0,3,0,0,0,19,0,0,0, + 67,0,0,0,115,123,0,0,0,124,1,0,100,1,0,107, + 2,0,114,53,0,121,16,0,116,0,0,106,1,0,131,0, + 0,125,1,0,87,110,22,0,4,116,2,0,107,10,0,114, + 52,0,1,1,1,100,2,0,83,89,110,1,0,88,121,17, + 0,116,3,0,106,4,0,124,1,0,25,125,2,0,87,110, + 46,0,4,116,5,0,107,10,0,114,118,0,1,1,1,124, + 0,0,106,6,0,124,1,0,131,1,0,125,2,0,124,2, + 0,116,3,0,106,4,0,124,1,0,60,89,110,1,0,88, + 124,2,0,83,41,3,122,210,71,101,116,32,116,104,101,32, + 102,105,110,100,101,114,32,102,111,114,32,116,104,101,32,112, + 97,116,104,32,101,110,116,114,121,32,102,114,111,109,32,115, + 121,115,46,112,97,116,104,95,105,109,112,111,114,116,101,114, + 95,99,97,99,104,101,46,10,10,32,32,32,32,32,32,32, + 32,73,102,32,116,104,101,32,112,97,116,104,32,101,110,116, + 114,121,32,105,115,32,110,111,116,32,105,110,32,116,104,101, + 32,99,97,99,104,101,44,32,102,105,110,100,32,116,104,101, + 32,97,112,112,114,111,112,114,105,97,116,101,32,102,105,110, + 100,101,114,10,32,32,32,32,32,32,32,32,97,110,100,32, + 99,97,99,104,101,32,105,116,46,32,73,102,32,110,111,32, + 102,105,110,100,101,114,32,105,115,32,97,118,97,105,108,97, + 98,108,101,44,32,115,116,111,114,101,32,78,111,110,101,46, + 10,10,32,32,32,32,32,32,32,32,114,30,0,0,0,78, + 41,7,114,3,0,0,0,114,45,0,0,0,218,17,70,105, + 108,101,78,111,116,70,111,117,110,100,69,114,114,111,114,114, + 7,0,0,0,114,245,0,0,0,114,131,0,0,0,114,249, + 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, + 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,34,4,0,0,115,22,0, + 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, + 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, + 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,99,3,0,0,0,0,0, + 0,0,6,0,0,0,3,0,0,0,67,0,0,0,115,119, + 0,0,0,116,0,0,124,2,0,100,1,0,131,2,0,114, + 39,0,124,2,0,106,1,0,124,1,0,131,1,0,92,2, + 0,125,3,0,125,4,0,110,21,0,124,2,0,106,2,0, + 124,1,0,131,1,0,125,3,0,103,0,0,125,4,0,124, + 3,0,100,0,0,107,9,0,114,88,0,116,3,0,106,4, + 0,124,1,0,124,3,0,131,2,0,83,116,3,0,106,5, + 0,124,1,0,100,0,0,131,2,0,125,5,0,124,4,0, + 124,5,0,95,6,0,124,5,0,83,41,2,78,114,117,0, + 0,0,41,7,114,108,0,0,0,114,117,0,0,0,114,176, + 0,0,0,114,114,0,0,0,114,173,0,0,0,114,154,0, + 0,0,114,150,0,0,0,41,6,114,164,0,0,0,114,119, + 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, + 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, + 103,101,116,95,115,112,101,99,56,4,0,0,115,18,0,0, + 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, + 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, + 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, + 78,99,4,0,0,0,0,0,0,0,9,0,0,0,5,0, + 0,0,67,0,0,0,115,243,0,0,0,103,0,0,125,4, + 0,120,230,0,124,2,0,68,93,191,0,125,5,0,116,0, + 0,124,5,0,116,1,0,116,2,0,102,2,0,131,2,0, + 115,43,0,113,13,0,124,0,0,106,3,0,124,5,0,131, + 1,0,125,6,0,124,6,0,100,1,0,107,9,0,114,13, + 0,116,4,0,124,6,0,100,2,0,131,2,0,114,106,0, + 124,6,0,106,5,0,124,1,0,124,3,0,131,2,0,125, + 7,0,110,18,0,124,0,0,106,6,0,124,1,0,124,6, + 0,131,2,0,125,7,0,124,7,0,100,1,0,107,8,0, + 114,139,0,113,13,0,124,7,0,106,7,0,100,1,0,107, + 9,0,114,158,0,124,7,0,83,124,7,0,106,8,0,125, + 8,0,124,8,0,100,1,0,107,8,0,114,191,0,116,9, + 0,100,3,0,131,1,0,130,1,0,124,4,0,106,10,0, + 124,8,0,131,1,0,1,113,13,0,87,116,11,0,106,12, + 0,124,1,0,100,1,0,131,2,0,125,7,0,124,4,0, + 124,7,0,95,8,0,124,7,0,83,100,1,0,83,41,4, + 122,63,70,105,110,100,32,116,104,101,32,108,111,97,100,101, + 114,32,111,114,32,110,97,109,101,115,112,97,99,101,95,112, + 97,116,104,32,102,111,114,32,116,104,105,115,32,109,111,100, + 117,108,101,47,112,97,99,107,97,103,101,32,110,97,109,101, + 46,78,114,175,0,0,0,122,19,115,112,101,99,32,109,105, + 115,115,105,110,103,32,108,111,97,100,101,114,41,13,114,137, + 0,0,0,114,69,0,0,0,218,5,98,121,116,101,115,114, + 251,0,0,0,114,108,0,0,0,114,175,0,0,0,114,252, + 0,0,0,114,120,0,0,0,114,150,0,0,0,114,99,0, + 0,0,114,143,0,0,0,114,114,0,0,0,114,154,0,0, + 0,41,9,114,164,0,0,0,114,119,0,0,0,114,35,0, + 0,0,114,174,0,0,0,218,14,110,97,109,101,115,112,97, + 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, + 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, + 101,116,95,115,112,101,99,71,4,0,0,115,40,0,0,0, + 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, + 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, + 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, + 110,100,101,114,46,95,103,101,116,95,115,112,101,99,99,4, + 0,0,0,0,0,0,0,6,0,0,0,4,0,0,0,67, + 0,0,0,115,140,0,0,0,124,2,0,100,1,0,107,8, + 0,114,21,0,116,0,0,106,1,0,125,2,0,124,0,0, + 106,2,0,124,1,0,124,2,0,124,3,0,131,3,0,125, + 4,0,124,4,0,100,1,0,107,8,0,114,58,0,100,1, + 0,83,124,4,0,106,3,0,100,1,0,107,8,0,114,132, + 0,124,4,0,106,4,0,125,5,0,124,5,0,114,125,0, + 100,2,0,124,4,0,95,5,0,116,6,0,124,1,0,124, + 5,0,124,0,0,106,2,0,131,3,0,124,4,0,95,4, + 0,124,4,0,83,100,1,0,83,110,4,0,124,4,0,83, + 100,1,0,83,41,3,122,98,102,105,110,100,32,116,104,101, + 32,109,111,100,117,108,101,32,111,110,32,115,121,115,46,112, + 97,116,104,32,111,114,32,39,112,97,116,104,39,32,98,97, + 115,101,100,32,111,110,32,115,121,115,46,112,97,116,104,95, + 104,111,111,107,115,32,97,110,100,10,32,32,32,32,32,32, + 32,32,115,121,115,46,112,97,116,104,95,105,109,112,111,114, + 116,101,114,95,99,97,99,104,101,46,78,90,9,110,97,109, + 101,115,112,97,99,101,41,7,114,7,0,0,0,114,35,0, + 0,0,114,255,0,0,0,114,120,0,0,0,114,150,0,0, + 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, + 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, + 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,103, + 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, + 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, + 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, + 105,110,100,95,115,112,101,99,99,3,0,0,0,0,0,0, + 0,4,0,0,0,3,0,0,0,67,0,0,0,115,41,0, + 0,0,124,0,0,106,0,0,124,1,0,124,2,0,131,2, + 0,125,3,0,124,3,0,100,1,0,107,8,0,114,34,0, + 100,1,0,83,124,3,0,106,1,0,83,41,2,122,170,102, + 105,110,100,32,116,104,101,32,109,111,100,117,108,101,32,111, + 110,32,115,121,115,46,112,97,116,104,32,111,114,32,39,112, + 97,116,104,39,32,98,97,115,101,100,32,111,110,32,115,121, + 115,46,112,97,116,104,95,104,111,111,107,115,32,97,110,100, + 10,32,32,32,32,32,32,32,32,115,121,115,46,112,97,116, + 104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101, + 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, + 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, + 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, + 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, + 10,32,32,32,32,32,32,32,32,78,41,2,114,175,0,0, + 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, + 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, + 125,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, + 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, + 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, + 177,0,0,0,114,244,0,0,0,114,249,0,0,0,114,251, + 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, + 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, + 5,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, + 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, + 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, + 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, + 0,90,1,0,100,0,0,90,2,0,100,1,0,90,3,0, + 100,2,0,100,3,0,132,0,0,90,4,0,100,4,0,100, + 5,0,132,0,0,90,5,0,101,6,0,90,7,0,100,6, + 0,100,7,0,132,0,0,90,8,0,100,8,0,100,9,0, + 132,0,0,90,9,0,100,10,0,100,11,0,100,12,0,132, + 1,0,90,10,0,100,13,0,100,14,0,132,0,0,90,11, + 0,101,12,0,100,15,0,100,16,0,132,0,0,131,1,0, + 90,13,0,100,17,0,100,18,0,132,0,0,90,14,0,100, + 10,0,83,41,19,218,10,70,105,108,101,70,105,110,100,101, + 114,122,172,70,105,108,101,45,98,97,115,101,100,32,102,105, + 110,100,101,114,46,10,10,32,32,32,32,73,110,116,101,114, + 97,99,116,105,111,110,115,32,119,105,116,104,32,116,104,101, + 32,102,105,108,101,32,115,121,115,116,101,109,32,97,114,101, + 32,99,97,99,104,101,100,32,102,111,114,32,112,101,114,102, + 111,114,109,97,110,99,101,44,32,98,101,105,110,103,10,32, + 32,32,32,114,101,102,114,101,115,104,101,100,32,119,104,101, + 110,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32, + 116,104,101,32,102,105,110,100,101,114,32,105,115,32,104,97, + 110,100,108,105,110,103,32,104,97,115,32,98,101,101,110,32, + 109,111,100,105,102,105,101,100,46,10,10,32,32,32,32,99, + 2,0,0,0,0,0,0,0,5,0,0,0,5,0,0,0, + 7,0,0,0,115,122,0,0,0,103,0,0,125,3,0,120, + 52,0,124,2,0,68,93,44,0,92,2,0,137,0,0,125, + 4,0,124,3,0,106,0,0,135,0,0,102,1,0,100,1, + 0,100,2,0,134,0,0,124,4,0,68,131,1,0,131,1, + 0,1,113,13,0,87,124,3,0,124,0,0,95,1,0,124, + 1,0,112,79,0,100,3,0,124,0,0,95,2,0,100,6, + 0,124,0,0,95,3,0,116,4,0,131,0,0,124,0,0, + 95,5,0,116,4,0,131,0,0,124,0,0,95,6,0,100, + 5,0,83,41,7,122,154,73,110,105,116,105,97,108,105,122, + 101,32,119,105,116,104,32,116,104,101,32,112,97,116,104,32, + 116,111,32,115,101,97,114,99,104,32,111,110,32,97,110,100, + 32,97,32,118,97,114,105,97,98,108,101,32,110,117,109,98, + 101,114,32,111,102,10,32,32,32,32,32,32,32,32,50,45, + 116,117,112,108,101,115,32,99,111,110,116,97,105,110,105,110, + 103,32,116,104,101,32,108,111,97,100,101,114,32,97,110,100, + 32,116,104,101,32,102,105,108,101,32,115,117,102,102,105,120, + 101,115,32,116,104,101,32,108,111,97,100,101,114,10,32,32, + 32,32,32,32,32,32,114,101,99,111,103,110,105,122,101,115, + 46,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,51,0,0,0,115,27,0,0,0,124,0,0,93,17, + 0,125,1,0,124,1,0,136,0,0,102,2,0,86,1,113, + 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, + 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, + 154,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, - 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,13,0,0,0,100,3,0,124,0,0, - 95,0,0,100,2,0,83,41,4,122,31,73,110,118,97,108, - 105,100,97,116,101,32,116,104,101,32,100,105,114,101,99,116, - 111,114,121,32,109,116,105,109,101,46,114,29,0,0,0,78, - 114,87,0,0,0,41,1,114,2,1,0,0,41,1,114,100, + 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, + 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, + 0,0,0,41,7,114,143,0,0,0,218,8,95,108,111,97, + 100,101,114,115,114,35,0,0,0,218,11,95,112,97,116,104, + 95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97, + 116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120, + 101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114, + 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, + 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, + 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, + 0,0,0,114,5,0,0,0,114,179,0,0,0,148,4,0, + 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, + 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, + 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, + 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, + 115,13,0,0,0,100,3,0,124,0,0,95,0,0,100,2, + 0,83,41,4,122,31,73,110,118,97,108,105,100,97,116,101, + 32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,109, + 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, + 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, + 0,0,162,4,0,0,115,2,0,0,0,0,2,122,28,70, + 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, + 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, + 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, + 115,59,0,0,0,124,0,0,106,0,0,124,1,0,131,1, + 0,125,2,0,124,2,0,100,1,0,107,8,0,114,37,0, + 100,1,0,103,0,0,102,2,0,83,124,2,0,106,1,0, + 124,2,0,106,2,0,112,55,0,103,0,0,102,2,0,83, + 41,2,122,197,84,114,121,32,116,111,32,102,105,110,100,32, + 97,32,108,111,97,100,101,114,32,102,111,114,32,116,104,101, + 32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108, + 101,44,32,111,114,32,116,104,101,32,110,97,109,101,115,112, + 97,99,101,10,32,32,32,32,32,32,32,32,112,97,99,107, + 97,103,101,32,112,111,114,116,105,111,110,115,46,32,82,101, + 116,117,114,110,115,32,40,108,111,97,100,101,114,44,32,108, + 105,115,116,45,111,102,45,112,111,114,116,105,111,110,115,41, + 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, + 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, + 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, + 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, + 10,32,32,32,32,32,32,32,32,78,41,3,114,175,0,0, + 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, + 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, + 168,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, + 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, + 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, + 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, + 0,0,0,124,1,0,124,2,0,124,3,0,131,2,0,125, + 6,0,116,0,0,124,2,0,124,3,0,100,1,0,124,6, + 0,100,2,0,124,4,0,131,2,2,83,41,3,78,114,120, + 0,0,0,114,150,0,0,0,41,1,114,161,0,0,0,41, + 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, + 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, + 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,255,0,0,0,180,4,0,0,115,6,0, + 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, + 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, + 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, + 67,0,0,0,115,228,1,0,0,100,1,0,125,3,0,124, + 1,0,106,0,0,100,2,0,131,1,0,100,3,0,25,125, + 4,0,121,34,0,116,1,0,124,0,0,106,2,0,112,49, + 0,116,3,0,106,4,0,131,0,0,131,1,0,106,5,0, + 125,5,0,87,110,24,0,4,116,6,0,107,10,0,114,85, + 0,1,1,1,100,10,0,125,5,0,89,110,1,0,88,124, + 5,0,124,0,0,106,7,0,107,3,0,114,120,0,124,0, + 0,106,8,0,131,0,0,1,124,5,0,124,0,0,95,7, + 0,116,9,0,131,0,0,114,153,0,124,0,0,106,10,0, + 125,6,0,124,4,0,106,11,0,131,0,0,125,7,0,110, + 15,0,124,0,0,106,12,0,125,6,0,124,4,0,125,7, + 0,124,7,0,124,6,0,107,6,0,114,45,1,116,13,0, + 124,0,0,106,2,0,124,4,0,131,2,0,125,8,0,120, + 100,0,124,0,0,106,14,0,68,93,77,0,92,2,0,125, + 9,0,125,10,0,100,5,0,124,9,0,23,125,11,0,116, + 13,0,124,8,0,124,11,0,131,2,0,125,12,0,116,15, + 0,124,12,0,131,1,0,114,208,0,124,0,0,106,16,0, + 124,10,0,124,1,0,124,12,0,124,8,0,103,1,0,124, + 2,0,131,5,0,83,113,208,0,87,116,17,0,124,8,0, + 131,1,0,125,3,0,120,120,0,124,0,0,106,14,0,68, + 93,109,0,92,2,0,125,9,0,125,10,0,116,13,0,124, + 0,0,106,2,0,124,4,0,124,9,0,23,131,2,0,125, + 12,0,116,18,0,106,19,0,100,6,0,124,12,0,100,7, + 0,100,3,0,131,2,1,1,124,7,0,124,9,0,23,124, + 6,0,107,6,0,114,55,1,116,15,0,124,12,0,131,1, + 0,114,55,1,124,0,0,106,16,0,124,10,0,124,1,0, + 124,12,0,100,8,0,124,2,0,131,5,0,83,113,55,1, + 87,124,3,0,114,224,1,116,18,0,106,19,0,100,9,0, + 124,8,0,131,2,0,1,116,18,0,106,20,0,124,1,0, + 100,8,0,131,2,0,125,13,0,124,8,0,103,1,0,124, + 13,0,95,21,0,124,13,0,83,100,8,0,83,41,11,122, + 125,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108, + 111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112, + 101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32, + 111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101, + 10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101, + 32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114, + 110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116, + 45,111,102,45,112,111,114,116,105,111,110,115,41,46,70,114, + 58,0,0,0,114,56,0,0,0,114,29,0,0,0,114,179, + 0,0,0,122,9,116,114,121,105,110,103,32,123,125,90,9, + 118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,115, + 105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,102, + 111,114,32,123,125,114,87,0,0,0,41,22,114,32,0,0, + 0,114,39,0,0,0,114,35,0,0,0,114,3,0,0,0, + 114,45,0,0,0,114,213,0,0,0,114,40,0,0,0,114, + 2,1,0,0,218,11,95,102,105,108,108,95,99,97,99,104, + 101,114,6,0,0,0,114,5,1,0,0,114,88,0,0,0, + 114,4,1,0,0,114,28,0,0,0,114,1,1,0,0,114, + 44,0,0,0,114,255,0,0,0,114,46,0,0,0,114,114, + 0,0,0,114,129,0,0,0,114,154,0,0,0,114,150,0, + 0,0,41,14,114,100,0,0,0,114,119,0,0,0,114,174, + 0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,99, + 101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,126, + 0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,104, + 101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,112, + 97,116,104,114,219,0,0,0,114,159,0,0,0,90,13,105, + 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, + 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, + 185,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, + 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, + 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, + 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, + 1,15,1,14,1,6,1,16,1,18,1,12,1,4,1,122, + 20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, + 95,115,112,101,99,99,1,0,0,0,0,0,0,0,9,0, + 0,0,13,0,0,0,67,0,0,0,115,11,1,0,0,124, + 0,0,106,0,0,125,1,0,121,31,0,116,1,0,106,2, + 0,124,1,0,112,33,0,116,1,0,106,3,0,131,0,0, + 131,1,0,125,2,0,87,110,33,0,4,116,4,0,116,5, + 0,116,6,0,102,3,0,107,10,0,114,75,0,1,1,1, + 103,0,0,125,2,0,89,110,1,0,88,116,7,0,106,8, + 0,106,9,0,100,1,0,131,1,0,115,112,0,116,10,0, + 124,2,0,131,1,0,124,0,0,95,11,0,110,111,0,116, + 10,0,131,0,0,125,3,0,120,90,0,124,2,0,68,93, + 82,0,125,4,0,124,4,0,106,12,0,100,2,0,131,1, + 0,92,3,0,125,5,0,125,6,0,125,7,0,124,6,0, + 114,191,0,100,3,0,106,13,0,124,5,0,124,7,0,106, + 14,0,131,0,0,131,2,0,125,8,0,110,6,0,124,5, + 0,125,8,0,124,3,0,106,15,0,124,8,0,131,1,0, + 1,113,128,0,87,124,3,0,124,0,0,95,11,0,116,7, + 0,106,8,0,106,9,0,116,16,0,131,1,0,114,7,1, + 100,4,0,100,5,0,132,0,0,124,2,0,68,131,1,0, + 124,0,0,95,17,0,100,6,0,83,41,7,122,68,70,105, + 108,108,32,116,104,101,32,99,97,99,104,101,32,111,102,32, + 112,111,116,101,110,116,105,97,108,32,109,111,100,117,108,101, + 115,32,97,110,100,32,112,97,99,107,97,103,101,115,32,102, + 111,114,32,116,104,105,115,32,100,105,114,101,99,116,111,114, + 121,46,114,0,0,0,0,114,58,0,0,0,122,5,123,125, + 46,123,125,99,1,0,0,0,0,0,0,0,2,0,0,0, + 3,0,0,0,83,0,0,0,115,28,0,0,0,104,0,0, + 124,0,0,93,18,0,125,1,0,124,1,0,106,0,0,131, + 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, + 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, + 9,60,115,101,116,99,111,109,112,62,4,5,0,0,115,2, + 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, + 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, + 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, + 78,41,18,114,35,0,0,0,114,3,0,0,0,90,7,108, + 105,115,116,100,105,114,114,45,0,0,0,114,250,0,0,0, + 218,15,80,101,114,109,105,115,115,105,111,110,69,114,114,111, + 114,218,18,78,111,116,65,68,105,114,101,99,116,111,114,121, + 69,114,114,111,114,114,7,0,0,0,114,8,0,0,0,114, + 9,0,0,0,114,3,1,0,0,114,4,1,0,0,114,83, + 0,0,0,114,47,0,0,0,114,88,0,0,0,218,3,97, + 100,100,114,10,0,0,0,114,5,1,0,0,41,9,114,100, + 0,0,0,114,35,0,0,0,90,8,99,111,110,116,101,110, + 116,115,90,21,108,111,119,101,114,95,115,117,102,102,105,120, + 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, + 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, + 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,114,7,1,0,0,231,4,0,0,115, + 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, + 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, + 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, + 101,114,46,95,102,105,108,108,95,99,97,99,104,101,99,1, + 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,7, + 0,0,0,115,25,0,0,0,135,0,0,135,1,0,102,2, + 0,100,1,0,100,2,0,134,0,0,125,2,0,124,2,0, + 83,41,3,97,20,1,0,0,65,32,99,108,97,115,115,32, + 109,101,116,104,111,100,32,119,104,105,99,104,32,114,101,116, + 117,114,110,115,32,97,32,99,108,111,115,117,114,101,32,116, + 111,32,117,115,101,32,111,110,32,115,121,115,46,112,97,116, + 104,95,104,111,111,107,10,32,32,32,32,32,32,32,32,119, + 104,105,99,104,32,119,105,108,108,32,114,101,116,117,114,110, + 32,97,110,32,105,110,115,116,97,110,99,101,32,117,115,105, + 110,103,32,116,104,101,32,115,112,101,99,105,102,105,101,100, + 32,108,111,97,100,101,114,115,32,97,110,100,32,116,104,101, + 32,112,97,116,104,10,32,32,32,32,32,32,32,32,99,97, + 108,108,101,100,32,111,110,32,116,104,101,32,99,108,111,115, + 117,114,101,46,10,10,32,32,32,32,32,32,32,32,73,102, + 32,116,104,101,32,112,97,116,104,32,99,97,108,108,101,100, + 32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,32, + 105,115,32,110,111,116,32,97,32,100,105,114,101,99,116,111, + 114,121,44,32,73,109,112,111,114,116,69,114,114,111,114,32, + 105,115,10,32,32,32,32,32,32,32,32,114,97,105,115,101, + 100,46,10,10,32,32,32,32,32,32,32,32,99,1,0,0, + 0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0, + 0,115,43,0,0,0,116,0,0,124,0,0,131,1,0,115, + 30,0,116,1,0,100,1,0,100,2,0,124,0,0,131,1, + 1,130,1,0,136,0,0,124,0,0,136,1,0,140,1,0, + 83,41,3,122,45,80,97,116,104,32,104,111,111,107,32,102, + 111,114,32,105,109,112,111,114,116,108,105,98,46,109,97,99, + 104,105,110,101,114,121,46,70,105,108,101,70,105,110,100,101, + 114,46,122,30,111,110,108,121,32,100,105,114,101,99,116,111, + 114,105,101,115,32,97,114,101,32,115,117,112,112,111,114,116, + 101,100,114,35,0,0,0,41,2,114,46,0,0,0,114,99, + 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, + 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, + 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, + 70,105,108,101,70,105,110,100,101,114,16,5,0,0,115,6, + 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, + 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, + 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, + 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, + 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, + 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, + 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, + 97,116,104,95,104,111,111,107,6,5,0,0,115,4,0,0, + 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, + 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, + 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, + 115,16,0,0,0,100,1,0,106,0,0,124,0,0,106,1, + 0,131,1,0,83,41,2,78,122,16,70,105,108,101,70,105, + 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, + 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, + 0,24,5,0,0,115,2,0,0,0,0,1,122,19,70,105, + 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, + 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, + 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, + 0,114,123,0,0,0,114,176,0,0,0,114,117,0,0,0, + 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, + 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,244,0,0,0,161,4,0,0,115,2,0,0,0, - 0,2,122,28,70,105,108,101,70,105,110,100,101,114,46,105, - 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, - 99,2,0,0,0,0,0,0,0,3,0,0,0,2,0,0, - 0,67,0,0,0,115,59,0,0,0,124,0,0,106,0,0, - 124,1,0,131,1,0,125,2,0,124,2,0,100,1,0,107, - 8,0,114,37,0,100,1,0,103,0,0,102,2,0,83,124, - 2,0,106,1,0,124,2,0,106,2,0,112,55,0,103,0, - 0,102,2,0,83,41,2,122,197,84,114,121,32,116,111,32, - 102,105,110,100,32,97,32,108,111,97,100,101,114,32,102,111, - 114,32,116,104,101,32,115,112,101,99,105,102,105,101,100,32, - 109,111,100,117,108,101,44,32,111,114,32,116,104,101,32,110, - 97,109,101,115,112,97,99,101,10,32,32,32,32,32,32,32, - 32,112,97,99,107,97,103,101,32,112,111,114,116,105,111,110, - 115,46,32,82,101,116,117,114,110,115,32,40,108,111,97,100, - 101,114,44,32,108,105,115,116,45,111,102,45,112,111,114,116, - 105,111,110,115,41,46,10,10,32,32,32,32,32,32,32,32, - 84,104,105,115,32,109,101,116,104,111,100,32,105,115,32,100, - 101,112,114,101,99,97,116,101,100,46,32,32,85,115,101,32, - 102,105,110,100,95,115,112,101,99,40,41,32,105,110,115,116, - 101,97,100,46,10,10,32,32,32,32,32,32,32,32,78,41, - 3,114,175,0,0,0,114,120,0,0,0,114,150,0,0,0, - 41,3,114,100,0,0,0,114,119,0,0,0,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,117,0,0,0,167,4,0,0,115,8,0,0,0,0,7, - 15,1,12,1,10,1,122,22,70,105,108,101,70,105,110,100, - 101,114,46,102,105,110,100,95,108,111,97,100,101,114,99,6, - 0,0,0,0,0,0,0,7,0,0,0,7,0,0,0,67, - 0,0,0,115,40,0,0,0,124,1,0,124,2,0,124,3, - 0,131,2,0,125,6,0,116,0,0,124,2,0,124,3,0, - 100,1,0,124,6,0,100,2,0,124,4,0,131,2,2,83, - 41,3,78,114,120,0,0,0,114,150,0,0,0,41,1,114, - 161,0,0,0,41,7,114,100,0,0,0,114,159,0,0,0, - 114,119,0,0,0,114,35,0,0,0,90,4,115,109,115,108, - 114,174,0,0,0,114,120,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,255,0,0,0,179,4, - 0,0,115,6,0,0,0,0,1,15,1,18,1,122,20,70, - 105,108,101,70,105,110,100,101,114,46,95,103,101,116,95,115, - 112,101,99,78,99,3,0,0,0,0,0,0,0,14,0,0, - 0,15,0,0,0,67,0,0,0,115,228,1,0,0,100,1, - 0,125,3,0,124,1,0,106,0,0,100,2,0,131,1,0, - 100,3,0,25,125,4,0,121,34,0,116,1,0,124,0,0, - 106,2,0,112,49,0,116,3,0,106,4,0,131,0,0,131, - 1,0,106,5,0,125,5,0,87,110,24,0,4,116,6,0, - 107,10,0,114,85,0,1,1,1,100,10,0,125,5,0,89, - 110,1,0,88,124,5,0,124,0,0,106,7,0,107,3,0, - 114,120,0,124,0,0,106,8,0,131,0,0,1,124,5,0, - 124,0,0,95,7,0,116,9,0,131,0,0,114,153,0,124, - 0,0,106,10,0,125,6,0,124,4,0,106,11,0,131,0, - 0,125,7,0,110,15,0,124,0,0,106,12,0,125,6,0, - 124,4,0,125,7,0,124,7,0,124,6,0,107,6,0,114, - 45,1,116,13,0,124,0,0,106,2,0,124,4,0,131,2, - 0,125,8,0,120,100,0,124,0,0,106,14,0,68,93,77, - 0,92,2,0,125,9,0,125,10,0,100,5,0,124,9,0, - 23,125,11,0,116,13,0,124,8,0,124,11,0,131,2,0, - 125,12,0,116,15,0,124,12,0,131,1,0,114,208,0,124, - 0,0,106,16,0,124,10,0,124,1,0,124,12,0,124,8, - 0,103,1,0,124,2,0,131,5,0,83,113,208,0,87,116, - 17,0,124,8,0,131,1,0,125,3,0,120,120,0,124,0, - 0,106,14,0,68,93,109,0,92,2,0,125,9,0,125,10, - 0,116,13,0,124,0,0,106,2,0,124,4,0,124,9,0, - 23,131,2,0,125,12,0,116,18,0,106,19,0,100,6,0, - 124,12,0,100,7,0,100,3,0,131,2,1,1,124,7,0, - 124,9,0,23,124,6,0,107,6,0,114,55,1,116,15,0, - 124,12,0,131,1,0,114,55,1,124,0,0,106,16,0,124, - 10,0,124,1,0,124,12,0,100,8,0,124,2,0,131,5, - 0,83,113,55,1,87,124,3,0,114,224,1,116,18,0,106, - 19,0,100,9,0,124,8,0,131,2,0,1,116,18,0,106, - 20,0,124,1,0,100,8,0,131,2,0,125,13,0,124,8, - 0,103,1,0,124,13,0,95,21,0,124,13,0,83,100,8, - 0,83,41,11,122,125,84,114,121,32,116,111,32,102,105,110, - 100,32,97,32,108,111,97,100,101,114,32,102,111,114,32,116, - 104,101,32,115,112,101,99,105,102,105,101,100,32,109,111,100, - 117,108,101,44,32,111,114,32,116,104,101,32,110,97,109,101, - 115,112,97,99,101,10,32,32,32,32,32,32,32,32,112,97, - 99,107,97,103,101,32,112,111,114,116,105,111,110,115,46,32, - 82,101,116,117,114,110,115,32,40,108,111,97,100,101,114,44, - 32,108,105,115,116,45,111,102,45,112,111,114,116,105,111,110, - 115,41,46,70,114,58,0,0,0,114,56,0,0,0,114,29, - 0,0,0,114,179,0,0,0,122,9,116,114,121,105,110,103, - 32,123,125,90,9,118,101,114,98,111,115,105,116,121,78,122, - 25,112,111,115,115,105,98,108,101,32,110,97,109,101,115,112, - 97,99,101,32,102,111,114,32,123,125,114,87,0,0,0,41, - 22,114,32,0,0,0,114,39,0,0,0,114,35,0,0,0, - 114,3,0,0,0,114,45,0,0,0,114,213,0,0,0,114, - 40,0,0,0,114,2,1,0,0,218,11,95,102,105,108,108, - 95,99,97,99,104,101,114,6,0,0,0,114,5,1,0,0, - 114,88,0,0,0,114,4,1,0,0,114,28,0,0,0,114, - 1,1,0,0,114,44,0,0,0,114,255,0,0,0,114,46, - 0,0,0,114,114,0,0,0,114,129,0,0,0,114,154,0, - 0,0,114,150,0,0,0,41,14,114,100,0,0,0,114,119, - 0,0,0,114,174,0,0,0,90,12,105,115,95,110,97,109, - 101,115,112,97,99,101,90,11,116,97,105,108,95,109,111,100, - 117,108,101,114,126,0,0,0,90,5,99,97,99,104,101,90, - 12,99,97,99,104,101,95,109,111,100,117,108,101,90,9,98, - 97,115,101,95,112,97,116,104,114,219,0,0,0,114,159,0, - 0,0,90,13,105,110,105,116,95,102,105,108,101,110,97,109, - 101,90,9,102,117,108,108,95,112,97,116,104,114,158,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,175,0,0,0,184,4,0,0,115,70,0,0,0,0,3, - 6,1,19,1,3,1,34,1,13,1,11,1,15,1,10,1, - 9,2,9,1,9,1,15,2,9,1,6,2,12,1,18,1, - 22,1,10,1,15,1,12,1,32,4,12,2,22,1,22,1, - 22,1,16,1,12,1,15,1,14,1,6,1,16,1,18,1, - 12,1,4,1,122,20,70,105,108,101,70,105,110,100,101,114, - 46,102,105,110,100,95,115,112,101,99,99,1,0,0,0,0, - 0,0,0,9,0,0,0,13,0,0,0,67,0,0,0,115, - 11,1,0,0,124,0,0,106,0,0,125,1,0,121,31,0, - 116,1,0,106,2,0,124,1,0,112,33,0,116,1,0,106, - 3,0,131,0,0,131,1,0,125,2,0,87,110,33,0,4, - 116,4,0,116,5,0,116,6,0,102,3,0,107,10,0,114, - 75,0,1,1,1,103,0,0,125,2,0,89,110,1,0,88, - 116,7,0,106,8,0,106,9,0,100,1,0,131,1,0,115, - 112,0,116,10,0,124,2,0,131,1,0,124,0,0,95,11, - 0,110,111,0,116,10,0,131,0,0,125,3,0,120,90,0, - 124,2,0,68,93,82,0,125,4,0,124,4,0,106,12,0, - 100,2,0,131,1,0,92,3,0,125,5,0,125,6,0,125, - 7,0,124,6,0,114,191,0,100,3,0,106,13,0,124,5, - 0,124,7,0,106,14,0,131,0,0,131,2,0,125,8,0, - 110,6,0,124,5,0,125,8,0,124,3,0,106,15,0,124, - 8,0,131,1,0,1,113,128,0,87,124,3,0,124,0,0, - 95,11,0,116,7,0,106,8,0,106,9,0,116,16,0,131, - 1,0,114,7,1,100,4,0,100,5,0,132,0,0,124,2, - 0,68,131,1,0,124,0,0,95,17,0,100,6,0,83,41, - 7,122,68,70,105,108,108,32,116,104,101,32,99,97,99,104, - 101,32,111,102,32,112,111,116,101,110,116,105,97,108,32,109, - 111,100,117,108,101,115,32,97,110,100,32,112,97,99,107,97, - 103,101,115,32,102,111,114,32,116,104,105,115,32,100,105,114, - 101,99,116,111,114,121,46,114,0,0,0,0,114,58,0,0, - 0,122,5,123,125,46,123,125,99,1,0,0,0,0,0,0, - 0,2,0,0,0,3,0,0,0,83,0,0,0,115,28,0, - 0,0,104,0,0,124,0,0,93,18,0,125,1,0,124,1, - 0,106,0,0,131,0,0,146,2,0,113,6,0,83,114,4, - 0,0,0,41,1,114,88,0,0,0,41,2,114,22,0,0, - 0,90,2,102,110,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,250,9,60,115,101,116,99,111,109,112,62,3, - 5,0,0,115,2,0,0,0,9,0,122,41,70,105,108,101, - 70,105,110,100,101,114,46,95,102,105,108,108,95,99,97,99, - 104,101,46,60,108,111,99,97,108,115,62,46,60,115,101,116, - 99,111,109,112,62,78,41,18,114,35,0,0,0,114,3,0, - 0,0,90,7,108,105,115,116,100,105,114,114,45,0,0,0, - 114,250,0,0,0,218,15,80,101,114,109,105,115,115,105,111, - 110,69,114,114,111,114,218,18,78,111,116,65,68,105,114,101, - 99,116,111,114,121,69,114,114,111,114,114,7,0,0,0,114, - 8,0,0,0,114,9,0,0,0,114,3,1,0,0,114,4, - 1,0,0,114,83,0,0,0,114,47,0,0,0,114,88,0, - 0,0,218,3,97,100,100,114,10,0,0,0,114,5,1,0, - 0,41,9,114,100,0,0,0,114,35,0,0,0,90,8,99, - 111,110,116,101,110,116,115,90,21,108,111,119,101,114,95,115, - 117,102,102,105,120,95,99,111,110,116,101,110,116,115,114,239, - 0,0,0,114,98,0,0,0,114,231,0,0,0,114,219,0, - 0,0,90,8,110,101,119,95,110,97,109,101,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,7,1,0,0, - 230,4,0,0,115,34,0,0,0,0,2,9,1,3,1,31, - 1,22,3,11,3,18,1,18,7,9,1,13,1,24,1,6, - 1,27,2,6,1,17,1,9,1,18,1,122,22,70,105,108, - 101,70,105,110,100,101,114,46,95,102,105,108,108,95,99,97, - 99,104,101,99,1,0,0,0,0,0,0,0,3,0,0,0, - 3,0,0,0,7,0,0,0,115,25,0,0,0,135,0,0, - 135,1,0,102,2,0,100,1,0,100,2,0,134,0,0,125, - 2,0,124,2,0,83,41,3,97,20,1,0,0,65,32,99, - 108,97,115,115,32,109,101,116,104,111,100,32,119,104,105,99, - 104,32,114,101,116,117,114,110,115,32,97,32,99,108,111,115, - 117,114,101,32,116,111,32,117,115,101,32,111,110,32,115,121, - 115,46,112,97,116,104,95,104,111,111,107,10,32,32,32,32, - 32,32,32,32,119,104,105,99,104,32,119,105,108,108,32,114, - 101,116,117,114,110,32,97,110,32,105,110,115,116,97,110,99, - 101,32,117,115,105,110,103,32,116,104,101,32,115,112,101,99, - 105,102,105,101,100,32,108,111,97,100,101,114,115,32,97,110, - 100,32,116,104,101,32,112,97,116,104,10,32,32,32,32,32, - 32,32,32,99,97,108,108,101,100,32,111,110,32,116,104,101, - 32,99,108,111,115,117,114,101,46,10,10,32,32,32,32,32, - 32,32,32,73,102,32,116,104,101,32,112,97,116,104,32,99, - 97,108,108,101,100,32,111,110,32,116,104,101,32,99,108,111, - 115,117,114,101,32,105,115,32,110,111,116,32,97,32,100,105, - 114,101,99,116,111,114,121,44,32,73,109,112,111,114,116,69, - 114,114,111,114,32,105,115,10,32,32,32,32,32,32,32,32, - 114,97,105,115,101,100,46,10,10,32,32,32,32,32,32,32, - 32,99,1,0,0,0,0,0,0,0,1,0,0,0,4,0, - 0,0,19,0,0,0,115,43,0,0,0,116,0,0,124,0, - 0,131,1,0,115,30,0,116,1,0,100,1,0,100,2,0, - 124,0,0,131,1,1,130,1,0,136,0,0,124,0,0,136, - 1,0,140,1,0,83,41,3,122,45,80,97,116,104,32,104, - 111,111,107,32,102,111,114,32,105,109,112,111,114,116,108,105, - 98,46,109,97,99,104,105,110,101,114,121,46,70,105,108,101, - 70,105,110,100,101,114,46,122,30,111,110,108,121,32,100,105, - 114,101,99,116,111,114,105,101,115,32,97,114,101,32,115,117, - 112,112,111,114,116,101,100,114,35,0,0,0,41,2,114,46, - 0,0,0,114,99,0,0,0,41,1,114,35,0,0,0,41, - 2,114,164,0,0,0,114,6,1,0,0,114,4,0,0,0, - 114,5,0,0,0,218,24,112,97,116,104,95,104,111,111,107, - 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,15, - 5,0,0,115,6,0,0,0,0,2,12,1,18,1,122,54, - 70,105,108,101,70,105,110,100,101,114,46,112,97,116,104,95, - 104,111,111,107,46,60,108,111,99,97,108,115,62,46,112,97, - 116,104,95,104,111,111,107,95,102,111,114,95,70,105,108,101, - 70,105,110,100,101,114,114,4,0,0,0,41,3,114,164,0, - 0,0,114,6,1,0,0,114,12,1,0,0,114,4,0,0, - 0,41,2,114,164,0,0,0,114,6,1,0,0,114,5,0, - 0,0,218,9,112,97,116,104,95,104,111,111,107,5,5,0, - 0,115,4,0,0,0,0,10,21,6,122,20,70,105,108,101, - 70,105,110,100,101,114,46,112,97,116,104,95,104,111,111,107, - 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, - 124,0,0,106,1,0,131,1,0,83,41,2,78,122,16,70, - 105,108,101,70,105,110,100,101,114,40,123,33,114,125,41,41, - 2,114,47,0,0,0,114,35,0,0,0,41,1,114,100,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,238,0,0,0,23,5,0,0,115,2,0,0,0,0, - 1,122,19,70,105,108,101,70,105,110,100,101,114,46,95,95, - 114,101,112,114,95,95,41,15,114,105,0,0,0,114,104,0, - 0,0,114,106,0,0,0,114,107,0,0,0,114,179,0,0, - 0,114,244,0,0,0,114,123,0,0,0,114,176,0,0,0, - 114,117,0,0,0,114,255,0,0,0,114,175,0,0,0,114, - 7,1,0,0,114,177,0,0,0,114,13,1,0,0,114,238, - 0,0,0,114,4,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,0,1,0,0,138,4,0,0, - 115,20,0,0,0,12,7,6,2,12,14,12,4,6,2,12, - 12,12,5,15,46,12,31,18,18,114,0,1,0,0,99,4, - 0,0,0,0,0,0,0,6,0,0,0,11,0,0,0,67, - 0,0,0,115,195,0,0,0,124,0,0,106,0,0,100,1, - 0,131,1,0,125,4,0,124,0,0,106,0,0,100,2,0, - 131,1,0,125,5,0,124,4,0,115,99,0,124,5,0,114, - 54,0,124,5,0,106,1,0,125,4,0,110,45,0,124,2, - 0,124,3,0,107,2,0,114,84,0,116,2,0,124,1,0, - 124,2,0,131,2,0,125,4,0,110,15,0,116,3,0,124, - 1,0,124,2,0,131,2,0,125,4,0,124,5,0,115,126, - 0,116,4,0,124,1,0,124,2,0,100,3,0,124,4,0, - 131,2,1,125,5,0,121,44,0,124,5,0,124,0,0,100, - 2,0,60,124,4,0,124,0,0,100,1,0,60,124,2,0, - 124,0,0,100,4,0,60,124,3,0,124,0,0,100,5,0, - 60,87,110,18,0,4,116,5,0,107,10,0,114,190,0,1, - 1,1,89,110,1,0,88,100,0,0,83,41,6,78,218,10, - 95,95,108,111,97,100,101,114,95,95,218,8,95,95,115,112, - 101,99,95,95,114,120,0,0,0,90,8,95,95,102,105,108, - 101,95,95,90,10,95,95,99,97,99,104,101,100,95,95,41, - 6,218,3,103,101,116,114,120,0,0,0,114,217,0,0,0, - 114,212,0,0,0,114,161,0,0,0,218,9,69,120,99,101, - 112,116,105,111,110,41,6,90,2,110,115,114,98,0,0,0, - 90,8,112,97,116,104,110,97,109,101,90,9,99,112,97,116, - 104,110,97,109,101,114,120,0,0,0,114,158,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,14, - 95,102,105,120,95,117,112,95,109,111,100,117,108,101,29,5, - 0,0,115,34,0,0,0,0,2,15,1,15,1,6,1,6, - 1,12,1,12,1,18,2,15,1,6,1,21,1,3,1,10, - 1,10,1,10,1,14,1,13,2,114,18,1,0,0,99,0, - 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,67, - 0,0,0,115,55,0,0,0,116,0,0,116,1,0,106,2, - 0,131,0,0,102,2,0,125,0,0,116,3,0,116,4,0, - 102,2,0,125,1,0,116,5,0,116,6,0,102,2,0,125, - 2,0,124,0,0,124,1,0,124,2,0,103,3,0,83,41, - 1,122,95,82,101,116,117,114,110,115,32,97,32,108,105,115, - 116,32,111,102,32,102,105,108,101,45,98,97,115,101,100,32, - 109,111,100,117,108,101,32,108,111,97,100,101,114,115,46,10, - 10,32,32,32,32,69,97,99,104,32,105,116,101,109,32,105, - 115,32,97,32,116,117,112,108,101,32,40,108,111,97,100,101, - 114,44,32,115,117,102,102,105,120,101,115,41,46,10,32,32, - 32,32,41,7,114,218,0,0,0,114,139,0,0,0,218,18, - 101,120,116,101,110,115,105,111,110,95,115,117,102,102,105,120, - 101,115,114,212,0,0,0,114,84,0,0,0,114,217,0,0, - 0,114,74,0,0,0,41,3,90,10,101,120,116,101,110,115, - 105,111,110,115,90,6,115,111,117,114,99,101,90,8,98,121, - 116,101,99,111,100,101,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,155,0,0,0,52,5,0,0,115,8, - 0,0,0,0,5,18,1,12,1,12,1,114,155,0,0,0, - 99,1,0,0,0,0,0,0,0,12,0,0,0,12,0,0, - 0,67,0,0,0,115,70,2,0,0,124,0,0,97,0,0, - 116,0,0,106,1,0,97,1,0,116,0,0,106,2,0,97, - 2,0,116,1,0,106,3,0,116,4,0,25,125,1,0,120, - 76,0,100,26,0,68,93,68,0,125,2,0,124,2,0,116, - 1,0,106,3,0,107,7,0,114,83,0,116,0,0,106,5, - 0,124,2,0,131,1,0,125,3,0,110,13,0,116,1,0, - 106,3,0,124,2,0,25,125,3,0,116,6,0,124,1,0, - 124,2,0,124,3,0,131,3,0,1,113,44,0,87,100,5, - 0,100,6,0,103,1,0,102,2,0,100,7,0,100,8,0, - 100,6,0,103,2,0,102,2,0,102,2,0,125,4,0,120, - 149,0,124,4,0,68,93,129,0,92,2,0,125,5,0,125, - 6,0,116,7,0,100,9,0,100,10,0,132,0,0,124,6, - 0,68,131,1,0,131,1,0,115,199,0,116,8,0,130,1, - 0,124,6,0,100,11,0,25,125,7,0,124,5,0,116,1, - 0,106,3,0,107,6,0,114,241,0,116,1,0,106,3,0, - 124,5,0,25,125,8,0,80,113,156,0,121,20,0,116,0, - 0,106,5,0,124,5,0,131,1,0,125,8,0,80,87,113, - 156,0,4,116,9,0,107,10,0,114,28,1,1,1,1,119, - 156,0,89,113,156,0,88,113,156,0,87,116,9,0,100,12, - 0,131,1,0,130,1,0,116,6,0,124,1,0,100,13,0, - 124,8,0,131,3,0,1,116,6,0,124,1,0,100,14,0, - 124,7,0,131,3,0,1,116,6,0,124,1,0,100,15,0, - 100,16,0,106,10,0,124,6,0,131,1,0,131,3,0,1, - 121,19,0,116,0,0,106,5,0,100,17,0,131,1,0,125, - 9,0,87,110,24,0,4,116,9,0,107,10,0,114,147,1, - 1,1,1,100,18,0,125,9,0,89,110,1,0,88,116,6, - 0,124,1,0,100,17,0,124,9,0,131,3,0,1,116,0, - 0,106,5,0,100,19,0,131,1,0,125,10,0,116,6,0, - 124,1,0,100,19,0,124,10,0,131,3,0,1,124,5,0, - 100,7,0,107,2,0,114,238,1,116,0,0,106,5,0,100, - 20,0,131,1,0,125,11,0,116,6,0,124,1,0,100,21, - 0,124,11,0,131,3,0,1,116,6,0,124,1,0,100,22, - 0,116,11,0,131,0,0,131,3,0,1,116,12,0,106,13, - 0,116,2,0,106,14,0,131,0,0,131,1,0,1,124,5, - 0,100,7,0,107,2,0,114,66,2,116,15,0,106,16,0, - 100,23,0,131,1,0,1,100,24,0,116,12,0,107,6,0, - 114,66,2,100,25,0,116,17,0,95,18,0,100,18,0,83, - 41,27,122,205,83,101,116,117,112,32,116,104,101,32,112,97, - 116,104,45,98,97,115,101,100,32,105,109,112,111,114,116,101, - 114,115,32,102,111,114,32,105,109,112,111,114,116,108,105,98, - 32,98,121,32,105,109,112,111,114,116,105,110,103,32,110,101, - 101,100,101,100,10,32,32,32,32,98,117,105,108,116,45,105, - 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, - 106,101,99,116,105,110,103,32,116,104,101,109,32,105,110,116, - 111,32,116,104,101,32,103,108,111,98,97,108,32,110,97,109, - 101,115,112,97,99,101,46,10,10,32,32,32,32,79,116,104, - 101,114,32,99,111,109,112,111,110,101,110,116,115,32,97,114, - 101,32,101,120,116,114,97,99,116,101,100,32,102,114,111,109, - 32,116,104,101,32,99,111,114,101,32,98,111,111,116,115,116, - 114,97,112,32,109,111,100,117,108,101,46,10,10,32,32,32, - 32,114,49,0,0,0,114,60,0,0,0,218,8,98,117,105, - 108,116,105,110,115,114,136,0,0,0,90,5,112,111,115,105, - 120,250,1,47,218,2,110,116,250,1,92,99,1,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,115,0,0,0, - 115,33,0,0,0,124,0,0,93,23,0,125,1,0,116,0, - 0,124,1,0,131,1,0,100,0,0,107,2,0,86,1,113, - 3,0,100,1,0,83,41,2,114,29,0,0,0,78,41,1, - 114,31,0,0,0,41,2,114,22,0,0,0,114,77,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,221,0,0,0,88,5,0,0,115,2,0,0,0,6,0, - 122,25,95,115,101,116,117,112,46,60,108,111,99,97,108,115, - 62,46,60,103,101,110,101,120,112,114,62,114,59,0,0,0, - 122,30,105,109,112,111,114,116,108,105,98,32,114,101,113,117, - 105,114,101,115,32,112,111,115,105,120,32,111,114,32,110,116, - 114,3,0,0,0,114,25,0,0,0,114,21,0,0,0,114, - 30,0,0,0,90,7,95,116,104,114,101,97,100,78,90,8, - 95,119,101,97,107,114,101,102,90,6,119,105,110,114,101,103, - 114,163,0,0,0,114,6,0,0,0,122,4,46,112,121,119, - 122,6,95,100,46,112,121,100,84,41,4,122,3,95,105,111, - 122,9,95,119,97,114,110,105,110,103,115,122,8,98,117,105, - 108,116,105,110,115,122,7,109,97,114,115,104,97,108,41,19, - 114,114,0,0,0,114,7,0,0,0,114,139,0,0,0,114, - 233,0,0,0,114,105,0,0,0,90,18,95,98,117,105,108, - 116,105,110,95,102,114,111,109,95,110,97,109,101,114,109,0, - 0,0,218,3,97,108,108,218,14,65,115,115,101,114,116,105, - 111,110,69,114,114,111,114,114,99,0,0,0,114,26,0,0, - 0,114,11,0,0,0,114,223,0,0,0,114,143,0,0,0, - 114,19,1,0,0,114,84,0,0,0,114,157,0,0,0,114, - 162,0,0,0,114,167,0,0,0,41,12,218,17,95,98,111, - 111,116,115,116,114,97,112,95,109,111,100,117,108,101,90,11, - 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, - 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, - 105,110,95,109,111,100,117,108,101,90,10,111,115,95,100,101, - 116,97,105,108,115,90,10,98,117,105,108,116,105,110,95,111, - 115,114,21,0,0,0,114,25,0,0,0,90,9,111,115,95, - 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, - 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, - 111,100,117,108,101,90,13,119,105,110,114,101,103,95,109,111, - 100,117,108,101,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,6,95,115,101,116,117,112,63,5,0,0,115, - 82,0,0,0,0,8,6,1,9,1,9,3,13,1,13,1, - 15,1,18,2,13,1,20,3,33,1,19,2,31,1,10,1, - 15,1,13,1,4,2,3,1,15,1,5,1,13,1,12,2, - 12,1,16,1,16,1,25,3,3,1,19,1,13,2,11,1, - 16,3,15,1,16,3,12,1,15,1,16,3,19,1,19,1, - 12,1,13,1,12,1,114,27,1,0,0,99,1,0,0,0, - 0,0,0,0,2,0,0,0,3,0,0,0,67,0,0,0, - 115,116,0,0,0,116,0,0,124,0,0,131,1,0,1,116, - 1,0,131,0,0,125,1,0,116,2,0,106,3,0,106,4, - 0,116,5,0,106,6,0,124,1,0,140,0,0,103,1,0, - 131,1,0,1,116,7,0,106,8,0,100,1,0,107,2,0, - 114,78,0,116,2,0,106,9,0,106,10,0,116,11,0,131, - 1,0,1,116,2,0,106,9,0,106,10,0,116,12,0,131, - 1,0,1,116,5,0,124,0,0,95,5,0,116,13,0,124, - 0,0,95,13,0,100,2,0,83,41,3,122,41,73,110,115, - 116,97,108,108,32,116,104,101,32,112,97,116,104,45,98,97, - 115,101,100,32,105,109,112,111,114,116,32,99,111,109,112,111, - 110,101,110,116,115,46,114,22,1,0,0,78,41,14,114,27, - 1,0,0,114,155,0,0,0,114,7,0,0,0,114,248,0, - 0,0,114,143,0,0,0,114,0,1,0,0,114,13,1,0, - 0,114,3,0,0,0,114,105,0,0,0,218,9,109,101,116, - 97,95,112,97,116,104,114,157,0,0,0,114,162,0,0,0, - 114,243,0,0,0,114,212,0,0,0,41,2,114,26,1,0, - 0,90,17,115,117,112,112,111,114,116,101,100,95,108,111,97, - 100,101,114,115,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,8,95,105,110,115,116,97,108,108,131,5,0, - 0,115,16,0,0,0,0,2,10,1,9,1,28,1,15,1, - 16,1,16,4,9,1,114,29,1,0,0,41,3,122,3,119, - 105,110,114,1,0,0,0,114,2,0,0,0,41,56,114,107, - 0,0,0,114,10,0,0,0,114,11,0,0,0,114,17,0, - 0,0,114,19,0,0,0,114,28,0,0,0,114,38,0,0, - 0,114,39,0,0,0,114,43,0,0,0,114,44,0,0,0, - 114,46,0,0,0,114,55,0,0,0,218,4,116,121,112,101, - 218,8,95,95,99,111,100,101,95,95,114,138,0,0,0,114, - 15,0,0,0,114,128,0,0,0,114,14,0,0,0,114,18, - 0,0,0,90,17,95,82,65,87,95,77,65,71,73,67,95, - 78,85,77,66,69,82,114,73,0,0,0,114,72,0,0,0, - 114,84,0,0,0,114,74,0,0,0,90,23,68,69,66,85, - 71,95,66,89,84,69,67,79,68,69,95,83,85,70,70,73, - 88,69,83,90,27,79,80,84,73,77,73,90,69,68,95,66, - 89,84,69,67,79,68,69,95,83,85,70,70,73,88,69,83, - 114,79,0,0,0,114,85,0,0,0,114,91,0,0,0,114, - 95,0,0,0,114,97,0,0,0,114,116,0,0,0,114,123, - 0,0,0,114,135,0,0,0,114,141,0,0,0,114,144,0, - 0,0,114,149,0,0,0,218,6,111,98,106,101,99,116,114, - 156,0,0,0,114,161,0,0,0,114,162,0,0,0,114,178, - 0,0,0,114,188,0,0,0,114,204,0,0,0,114,212,0, - 0,0,114,217,0,0,0,114,223,0,0,0,114,218,0,0, - 0,114,224,0,0,0,114,241,0,0,0,114,243,0,0,0, - 114,0,1,0,0,114,18,1,0,0,114,155,0,0,0,114, - 27,1,0,0,114,29,1,0,0,114,4,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,218,8,60, - 109,111,100,117,108,101,62,8,0,0,0,115,98,0,0,0, - 6,17,6,3,12,12,12,5,12,5,12,6,12,12,12,10, - 12,9,12,5,12,7,15,22,15,111,22,1,18,2,6,1, - 6,2,9,2,9,2,10,2,21,44,12,33,12,19,12,12, - 12,12,12,28,12,17,21,55,21,12,18,10,12,14,9,3, - 12,1,15,65,19,64,19,28,22,110,19,41,25,45,25,16, - 6,3,25,53,19,57,19,42,19,134,19,147,15,23,12,11, - 12,68, + 0,0,114,0,1,0,0,139,4,0,0,115,20,0,0,0, + 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, + 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, + 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, + 0,0,0,124,0,0,106,0,0,100,1,0,131,1,0,125, + 4,0,124,0,0,106,0,0,100,2,0,131,1,0,125,5, + 0,124,4,0,115,99,0,124,5,0,114,54,0,124,5,0, + 106,1,0,125,4,0,110,45,0,124,2,0,124,3,0,107, + 2,0,114,84,0,116,2,0,124,1,0,124,2,0,131,2, + 0,125,4,0,110,15,0,116,3,0,124,1,0,124,2,0, + 131,2,0,125,4,0,124,5,0,115,126,0,116,4,0,124, + 1,0,124,2,0,100,3,0,124,4,0,131,2,1,125,5, + 0,121,44,0,124,5,0,124,0,0,100,2,0,60,124,4, + 0,124,0,0,100,1,0,60,124,2,0,124,0,0,100,4, + 0,60,124,3,0,124,0,0,100,5,0,60,87,110,18,0, + 4,116,5,0,107,10,0,114,190,0,1,1,1,89,110,1, + 0,88,100,0,0,83,41,6,78,218,10,95,95,108,111,97, + 100,101,114,95,95,218,8,95,95,115,112,101,99,95,95,114, + 120,0,0,0,90,8,95,95,102,105,108,101,95,95,90,10, + 95,95,99,97,99,104,101,100,95,95,41,6,218,3,103,101, + 116,114,120,0,0,0,114,217,0,0,0,114,212,0,0,0, + 114,161,0,0,0,218,9,69,120,99,101,112,116,105,111,110, + 41,6,90,2,110,115,114,98,0,0,0,90,8,112,97,116, + 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, + 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, + 117,112,95,109,111,100,117,108,101,30,5,0,0,115,34,0, + 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, + 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, + 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, + 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,55, + 0,0,0,116,0,0,116,1,0,106,2,0,131,0,0,102, + 2,0,125,0,0,116,3,0,116,4,0,102,2,0,125,1, + 0,116,5,0,116,6,0,102,2,0,125,2,0,124,0,0, + 124,1,0,124,2,0,103,3,0,83,41,1,122,95,82,101, + 116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,32, + 102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,108, + 101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,32, + 69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,116, + 117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,117, + 102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,114, + 218,0,0,0,114,139,0,0,0,218,18,101,120,116,101,110, + 115,105,111,110,95,115,117,102,102,105,120,101,115,114,212,0, + 0,0,114,84,0,0,0,114,217,0,0,0,114,74,0,0, + 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, + 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,155,0,0,0,53,5,0,0,115,8,0,0,0,0,5, + 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, + 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, + 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, + 0,97,1,0,116,0,0,106,2,0,97,2,0,116,1,0, + 106,3,0,116,4,0,25,125,1,0,120,76,0,100,26,0, + 68,93,68,0,125,2,0,124,2,0,116,1,0,106,3,0, + 107,7,0,114,83,0,116,0,0,106,5,0,124,2,0,131, + 1,0,125,3,0,110,13,0,116,1,0,106,3,0,124,2, + 0,25,125,3,0,116,6,0,124,1,0,124,2,0,124,3, + 0,131,3,0,1,113,44,0,87,100,5,0,100,6,0,103, + 1,0,102,2,0,100,7,0,100,8,0,100,6,0,103,2, + 0,102,2,0,102,2,0,125,4,0,120,149,0,124,4,0, + 68,93,129,0,92,2,0,125,5,0,125,6,0,116,7,0, + 100,9,0,100,10,0,132,0,0,124,6,0,68,131,1,0, + 131,1,0,115,199,0,116,8,0,130,1,0,124,6,0,100, + 11,0,25,125,7,0,124,5,0,116,1,0,106,3,0,107, + 6,0,114,241,0,116,1,0,106,3,0,124,5,0,25,125, + 8,0,80,113,156,0,121,20,0,116,0,0,106,5,0,124, + 5,0,131,1,0,125,8,0,80,87,113,156,0,4,116,9, + 0,107,10,0,114,28,1,1,1,1,119,156,0,89,113,156, + 0,88,113,156,0,87,116,9,0,100,12,0,131,1,0,130, + 1,0,116,6,0,124,1,0,100,13,0,124,8,0,131,3, + 0,1,116,6,0,124,1,0,100,14,0,124,7,0,131,3, + 0,1,116,6,0,124,1,0,100,15,0,100,16,0,106,10, + 0,124,6,0,131,1,0,131,3,0,1,121,19,0,116,0, + 0,106,5,0,100,17,0,131,1,0,125,9,0,87,110,24, + 0,4,116,9,0,107,10,0,114,147,1,1,1,1,100,18, + 0,125,9,0,89,110,1,0,88,116,6,0,124,1,0,100, + 17,0,124,9,0,131,3,0,1,116,0,0,106,5,0,100, + 19,0,131,1,0,125,10,0,116,6,0,124,1,0,100,19, + 0,124,10,0,131,3,0,1,124,5,0,100,7,0,107,2, + 0,114,238,1,116,0,0,106,5,0,100,20,0,131,1,0, + 125,11,0,116,6,0,124,1,0,100,21,0,124,11,0,131, + 3,0,1,116,6,0,124,1,0,100,22,0,116,11,0,131, + 0,0,131,3,0,1,116,12,0,106,13,0,116,2,0,106, + 14,0,131,0,0,131,1,0,1,124,5,0,100,7,0,107, + 2,0,114,66,2,116,15,0,106,16,0,100,23,0,131,1, + 0,1,100,24,0,116,12,0,107,6,0,114,66,2,100,25, + 0,116,17,0,95,18,0,100,18,0,83,41,27,122,205,83, + 101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,97, + 115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,111, + 114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105, + 109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,10, + 32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,100, + 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, + 110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,101, + 32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99, + 101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,111, + 109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,116, + 114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,32, + 99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,109, + 111,100,117,108,101,46,10,10,32,32,32,32,114,49,0,0, + 0,114,60,0,0,0,218,8,98,117,105,108,116,105,110,115, + 114,136,0,0,0,90,5,112,111,115,105,120,250,1,47,218, + 2,110,116,250,1,92,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,115,0,0,0,115,33,0,0,0, + 124,0,0,93,23,0,125,1,0,116,0,0,124,1,0,131, + 1,0,100,0,0,107,2,0,86,1,113,3,0,100,1,0, + 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, + 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, + 89,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, + 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, + 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, + 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, + 112,111,115,105,120,32,111,114,32,110,116,114,3,0,0,0, + 114,25,0,0,0,114,21,0,0,0,114,30,0,0,0,90, + 7,95,116,104,114,101,97,100,78,90,8,95,119,101,97,107, + 114,101,102,90,6,119,105,110,114,101,103,114,163,0,0,0, + 114,6,0,0,0,122,4,46,112,121,119,122,6,95,100,46, + 112,121,100,84,41,4,122,3,95,105,111,122,9,95,119,97, + 114,110,105,110,103,115,122,8,98,117,105,108,116,105,110,115, + 122,7,109,97,114,115,104,97,108,41,19,114,114,0,0,0, + 114,7,0,0,0,114,139,0,0,0,114,233,0,0,0,114, + 105,0,0,0,90,18,95,98,117,105,108,116,105,110,95,102, + 114,111,109,95,110,97,109,101,114,109,0,0,0,218,3,97, + 108,108,218,14,65,115,115,101,114,116,105,111,110,69,114,114, + 111,114,114,99,0,0,0,114,26,0,0,0,114,11,0,0, + 0,114,223,0,0,0,114,143,0,0,0,114,19,1,0,0, + 114,84,0,0,0,114,157,0,0,0,114,162,0,0,0,114, + 167,0,0,0,41,12,218,17,95,98,111,111,116,115,116,114, + 97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,95, + 109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,95, + 110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,111, + 100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,115, + 90,10,98,117,105,108,116,105,110,95,111,115,114,21,0,0, + 0,114,25,0,0,0,90,9,111,115,95,109,111,100,117,108, + 101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,101, + 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, + 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, + 95,115,101,116,117,112,64,5,0,0,115,82,0,0,0,0, + 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, + 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, + 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, + 1,25,3,3,1,19,1,13,2,11,1,16,3,15,1,16, + 3,12,1,15,1,16,3,19,1,19,1,12,1,13,1,12, + 1,114,27,1,0,0,99,1,0,0,0,0,0,0,0,2, + 0,0,0,3,0,0,0,67,0,0,0,115,116,0,0,0, + 116,0,0,124,0,0,131,1,0,1,116,1,0,131,0,0, + 125,1,0,116,2,0,106,3,0,106,4,0,116,5,0,106, + 6,0,124,1,0,140,0,0,103,1,0,131,1,0,1,116, + 7,0,106,8,0,100,1,0,107,2,0,114,78,0,116,2, + 0,106,9,0,106,10,0,116,11,0,131,1,0,1,116,2, + 0,106,9,0,106,10,0,116,12,0,131,1,0,1,116,5, + 0,124,0,0,95,5,0,116,13,0,124,0,0,95,13,0, + 100,2,0,83,41,3,122,41,73,110,115,116,97,108,108,32, + 116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,105, + 109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,115, + 46,114,22,1,0,0,78,41,14,114,27,1,0,0,114,155, + 0,0,0,114,7,0,0,0,114,248,0,0,0,114,143,0, + 0,0,114,0,1,0,0,114,13,1,0,0,114,3,0,0, + 0,114,105,0,0,0,218,9,109,101,116,97,95,112,97,116, + 104,114,157,0,0,0,114,162,0,0,0,114,243,0,0,0, + 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, + 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, + 95,105,110,115,116,97,108,108,132,5,0,0,115,16,0,0, + 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, + 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, + 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,17,0,0,0,114,19,0, + 0,0,114,28,0,0,0,114,38,0,0,0,114,39,0,0, + 0,114,43,0,0,0,114,44,0,0,0,114,46,0,0,0, + 114,55,0,0,0,218,4,116,121,112,101,218,8,95,95,99, + 111,100,101,95,95,114,138,0,0,0,114,15,0,0,0,114, + 128,0,0,0,114,14,0,0,0,114,18,0,0,0,90,17, + 95,82,65,87,95,77,65,71,73,67,95,78,85,77,66,69, + 82,114,73,0,0,0,114,72,0,0,0,114,84,0,0,0, + 114,74,0,0,0,90,23,68,69,66,85,71,95,66,89,84, + 69,67,79,68,69,95,83,85,70,70,73,88,69,83,90,27, + 79,80,84,73,77,73,90,69,68,95,66,89,84,69,67,79, + 68,69,95,83,85,70,70,73,88,69,83,114,79,0,0,0, + 114,85,0,0,0,114,91,0,0,0,114,95,0,0,0,114, + 97,0,0,0,114,116,0,0,0,114,123,0,0,0,114,135, + 0,0,0,114,141,0,0,0,114,144,0,0,0,114,149,0, + 0,0,218,6,111,98,106,101,99,116,114,156,0,0,0,114, + 161,0,0,0,114,162,0,0,0,114,178,0,0,0,114,188, + 0,0,0,114,204,0,0,0,114,212,0,0,0,114,217,0, + 0,0,114,223,0,0,0,114,218,0,0,0,114,224,0,0, + 0,114,241,0,0,0,114,243,0,0,0,114,0,1,0,0, + 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, + 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, + 101,62,8,0,0,0,115,98,0,0,0,6,17,6,3,12, + 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, + 7,15,22,15,111,22,1,18,2,6,1,6,2,9,2,9, + 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, + 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, + 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, + 57,19,42,19,134,19,147,15,23,12,11,12,68, }; -- cgit v0.12 From bffa73e582c1a842d91d25ce93055f5e69074764 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Mon, 28 Dec 2015 21:51:02 -0800 Subject: Issue #25972, #20440: Fix compilation on Windows --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dddeba0..c538ec7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5150,7 +5150,7 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) return -1; Py_INCREF(args); - Py_SETREF((PyBaseExceptionObject *)self->args, args); + Py_SETREF(((PyBaseExceptionObject *)self)->args, args); return 0; } -- cgit v0.12 From 0d250bc119489fa7d094d4a3fd2fd2fa0a508145 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Dec 2015 22:34:23 +0200 Subject: Issue #25971: Optimized creating Fractions from floats by 2 times and from Decimals by 3 times. Unified error messages in float.as_integer_ratio(), Decimal.as_integer_ratio(), and Fraction constructors. --- Lib/_pydecimal.py | 6 ++---- Lib/fractions.py | 32 ++++---------------------------- Lib/test/test_fractions.py | 14 +++++++------- Misc/NEWS | 3 +++ Objects/floatobject.c | 12 ++++++------ 5 files changed, 22 insertions(+), 45 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index eb7bba8..02365ca 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1026,11 +1026,9 @@ class Decimal(object): """ if self._is_special: if self.is_nan(): - raise ValueError("Cannot pass NaN " - "to decimal.as_integer_ratio.") + raise ValueError("cannot convert NaN to integer ratio") else: - raise OverflowError("Cannot pass infinity " - "to decimal.as_integer_ratio.") + raise OverflowError("cannot convert Infinity to integer ratio") if not self: return 0, 1 diff --git a/Lib/fractions.py b/Lib/fractions.py index 60b0728..64d746b 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -125,17 +125,9 @@ class Fraction(numbers.Rational): self._denominator = numerator.denominator return self - elif isinstance(numerator, float): - # Exact conversion from float - value = Fraction.from_float(numerator) - self._numerator = value._numerator - self._denominator = value._denominator - return self - - elif isinstance(numerator, Decimal): - value = Fraction.from_decimal(numerator) - self._numerator = value._numerator - self._denominator = value._denominator + elif isinstance(numerator, (float, Decimal)): + # Exact conversion + self._numerator, self._denominator = numerator.as_integer_ratio() return self elif isinstance(numerator, str): @@ -210,10 +202,6 @@ class Fraction(numbers.Rational): elif not isinstance(f, float): raise TypeError("%s.from_float() only takes floats, not %r (%s)" % (cls.__name__, f, type(f).__name__)) - if math.isnan(f): - raise ValueError("Cannot convert %r to %s." % (f, cls.__name__)) - if math.isinf(f): - raise OverflowError("Cannot convert %r to %s." % (f, cls.__name__)) return cls(*f.as_integer_ratio()) @classmethod @@ -226,19 +214,7 @@ class Fraction(numbers.Rational): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) - if dec.is_infinite(): - raise OverflowError( - "Cannot convert %s to %s." % (dec, cls.__name__)) - if dec.is_nan(): - raise ValueError("Cannot convert %s to %s." % (dec, cls.__name__)) - sign, digits, exp = dec.as_tuple() - digits = int(''.join(map(str, digits))) - if sign: - digits = -digits - if exp >= 0: - return cls(digits * 10 ** exp) - else: - return cls(digits, 10 ** -exp) + return cls(*dec.as_integer_ratio()) def limit_denominator(self, max_denominator=1000000): """Closest Fraction to self with denominator at most max_denominator. diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 1699852..73d2dd3 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -263,13 +263,13 @@ class FractionTest(unittest.TestCase): nan = inf - inf # bug 16469: error types should be consistent with float -> int self.assertRaisesMessage( - OverflowError, "Cannot convert inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, inf) self.assertRaisesMessage( - OverflowError, "Cannot convert -inf to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_float, -inf) self.assertRaisesMessage( - ValueError, "Cannot convert nan to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_float, nan) def testFromDecimal(self): @@ -284,16 +284,16 @@ class FractionTest(unittest.TestCase): # bug 16469: error types should be consistent with decimal -> int self.assertRaisesMessage( - OverflowError, "Cannot convert Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("inf")) self.assertRaisesMessage( - OverflowError, "Cannot convert -Infinity to Fraction.", + OverflowError, "cannot convert Infinity to integer ratio", F.from_decimal, Decimal("-inf")) self.assertRaisesMessage( - ValueError, "Cannot convert NaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("nan")) self.assertRaisesMessage( - ValueError, "Cannot convert sNaN to Fraction.", + ValueError, "cannot convert NaN to integer ratio", F.from_decimal, Decimal("snan")) def testLimitDenominator(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6ec585a..6333401 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -126,6 +126,9 @@ Core and Builtins Library ------- +- Issue #25971: Optimized creating Fractions from floats by 2 times and from + Decimals by 3 times. + - Issue #25802: Document as deprecated the remaining implementations of importlib.abc.Loader.load_module(). diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d92bec3..2949174 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1466,14 +1466,14 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) CONVERT_TO_DOUBLE(v, self); if (Py_IS_INFINITY(self)) { - PyErr_SetString(PyExc_OverflowError, - "Cannot pass infinity to float.as_integer_ratio."); - return NULL; + PyErr_SetString(PyExc_OverflowError, + "cannot convert Infinity to integer ratio"); + return NULL; } if (Py_IS_NAN(self)) { - PyErr_SetString(PyExc_ValueError, - "Cannot pass NaN to float.as_integer_ratio."); - return NULL; + PyErr_SetString(PyExc_ValueError, + "cannot convert NaN to integer ratio"); + return NULL; } PyFPE_START_PROTECT("as_integer_ratio", goto error); -- cgit v0.12 From 4e6aad1f7ac4e91a5e6c44e8ab3a2105acd65d74 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 29 Dec 2015 22:55:48 +0200 Subject: Clean up float.as_integer_ratio(). --- Objects/floatobject.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 2949174..eb60659 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1451,18 +1451,12 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) int exponent; int i; - PyObject *prev; PyObject *py_exponent = NULL; PyObject *numerator = NULL; PyObject *denominator = NULL; PyObject *result_pair = NULL; PyNumberMethods *long_methods = PyLong_Type.tp_as_number; -#define INPLACE_UPDATE(obj, call) \ - prev = obj; \ - obj = call; \ - Py_DECREF(prev); \ - CONVERT_TO_DOUBLE(v, self); if (Py_IS_INFINITY(self)) { @@ -1489,29 +1483,31 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) to be truncated by PyLong_FromDouble(). */ numerator = PyLong_FromDouble(float_part); - if (numerator == NULL) goto error; + if (numerator == NULL) + goto error; + denominator = PyLong_FromLong(1); + if (denominator == NULL) + goto error; + py_exponent = PyLong_FromLong(Py_ABS(exponent)); + if (py_exponent == NULL) + goto error; /* fold in 2**exponent */ - denominator = PyLong_FromLong(1); - py_exponent = PyLong_FromLong(labs((long)exponent)); - if (py_exponent == NULL) goto error; - INPLACE_UPDATE(py_exponent, - long_methods->nb_lshift(denominator, py_exponent)); - if (py_exponent == NULL) goto error; if (exponent > 0) { - INPLACE_UPDATE(numerator, - long_methods->nb_multiply(numerator, py_exponent)); - if (numerator == NULL) goto error; + Py_SETREF(numerator, + long_methods->nb_lshift(numerator, py_exponent)); + if (numerator == NULL) + goto error; } else { - Py_DECREF(denominator); - denominator = py_exponent; - py_exponent = NULL; + Py_SETREF(denominator, + long_methods->nb_lshift(denominator, py_exponent)); + if (denominator == NULL) + goto error; } result_pair = PyTuple_Pack(2, numerator, denominator); -#undef INPLACE_UPDATE error: Py_XDECREF(py_exponent); Py_XDECREF(denominator); -- cgit v0.12 From 317f64f048b6da8a53870ce6a1d63ad458ece95f Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:18:34 -0500 Subject: #21815: violate IMAP RFC to be compatible with, e.g., gmail and others, including imaplib's own behavior. I'm applying this only to 3.6 because there's a potential backward compatibility concern: if there are servers that include ] characters in the 'text' portion of their imap responses, this code change could introduce a new bug. Patch by Lita Cho, reviewed by Jessica McKellar, Berker Peksag, Maciej Szulik, silentghost, and me (I fleshed out the comments with the additional info/concerns.) --- Doc/library/imaplib.rst | 11 +++++++++++ Lib/imaplib.py | 10 +++++++++- Lib/test/test_imaplib.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 4 ++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 15b0932..fbb802c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -500,6 +500,17 @@ An :class:`IMAP4` instance has the following methods: M.store(num, '+FLAGS', '\\Deleted') M.expunge() + .. note:: + + Creating flags containing ']' (for example: "[test]") violates + :rfc:`3501` (the IMAP protocol). However, imaplib has historically + allowed creation of such tags, and popular IMAP servers, such as Gmail, + accept and produce such flags. There are non-Python programs which also + create such tags. Although it is an RFC violation and IMAP clients and + servers are supposed to be strict, imaplib nontheless continues to allow + such tags to be created for backward compatibility reasons, and as of + python 3.5.2/3.6.0, handles them if they are sent from the server, since + this improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4e8a4bb..a63ba8d 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -111,7 +111,15 @@ InternalDate = re.compile(br'.*INTERNALDATE "' # Literal is no longer used; kept for backward compatibility. Literal = re.compile(br'.*{(?P\d+)}$', re.ASCII) MapCRLF = re.compile(br'\r\n|\r|\n') -Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P[^\]]*))?\]') +# We no longer exclude the ']' character from the data portion of the response +# code, even though it violates the RFC. Popular IMAP servers such as Gmail +# allow flags with ']', and there are programs (including imaplib!) that can +# produce them. The problem with this is if the 'text' portion of the response +# includes a ']' we'll parse the response wrong (which is the point of the RFC +# restriction). However, that seems less likely to be a problem in practice +# than being unable to correctly parse flags that include ']' chars, which +# was reported as a real-world problem in issue #21815. +Response_code = re.compile(br'\[(?P[A-Z-]+)( (?P.*))?\]') Untagged_response = re.compile(br'\* (?P[A-Z-]+)( (?P.*))?') # Untagged_status is no longer used; kept for backward compatibility Untagged_status = re.compile( diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 07157f5..8e4990b 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -243,6 +243,55 @@ class ThreadedNetworkedTests(unittest.TestCase): client.shutdown() @reap_threads + def test_bracket_flags(self): + + # This violates RFC 3501, which disallows ']' characters in tag names, + # but imaplib has allowed producing such tags forever, other programs + # also produce them (eg: OtherInbox's Organizer app as of 20140716), + # and Gmail, for example, accepts them and produces them. So we + # support them. See issue #21815. + + class BracketFlagHandler(SimpleIMAPHandler): + + def handle(self): + self.flags = ['Answered', 'Flagged', 'Deleted', 'Seen', 'Draft'] + super().handle() + + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + + def cmd_SELECT(self, tag, args): + flag_msg = ' \\'.join(self.flags) + self._send_line(('* FLAGS (%s)' % flag_msg).encode('ascii')) + self._send_line(b'* 2 EXISTS') + self._send_line(b'* 0 RECENT') + msg = ('* OK [PERMANENTFLAGS %s \\*)] Flags permitted.' + % flag_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + + def cmd_STORE(self, tag, args): + new_flags = args[2].strip('(').strip(')').split() + self.flags.extend(new_flags) + flags_msg = '(FLAGS (%s))' % ' \\'.join(self.flags) + msg = '* %s FETCH %s' % (args[0], flags_msg) + self._send_line(msg.encode('ascii')) + self._send_tagged(tag, 'OK', 'STORE completed.') + + with self.reaped_pair(BracketFlagHandler) as (server, client): + code, data = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(code, 'OK') + self.assertEqual(server.response, b'ZmFrZQ==\r\n') + client.select('test') + typ, [data] = client.store(b'1', "+FLAGS", "[test]") + self.assertIn(b'[test]', data) + client.select('test') + typ, [data] = client.response('PERMANENTFLAGS') + self.assertIn(b'[test]', data) + + @reap_threads def test_issue5949(self): class EOFHandler(socketserver.StreamRequestHandler): diff --git a/Misc/NEWS b/Misc/NEWS index 83c07f5..7277838 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,10 @@ Core and Builtins Library ------- +- Issue #21815: Accept ] characters in the data portion of imap responses, + in order to handle the flags with square brackets accepted and produced + by servers such as gmail. + - Issue #25447: fileinput now uses sys.stdin as-is if it does not have a buffer attribute (restores backward compatibility). -- cgit v0.12 From 75f104a74535ddcbc390c4e8362fd7157bace185 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sat, 2 Jan 2016 17:25:59 -0500 Subject: #21815: Make the doc change match what I actually did. --- Doc/library/imaplib.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index fbb802c..cd214ff 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -509,8 +509,8 @@ An :class:`IMAP4` instance has the following methods: create such tags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib nontheless continues to allow such tags to be created for backward compatibility reasons, and as of - python 3.5.2/3.6.0, handles them if they are sent from the server, since - this improves real-world compatibility. + python 3.6, handles them if they are sent from the server, since this + improves real-world compatibility. .. method:: IMAP4.subscribe(mailbox) -- cgit v0.12 From 576f132b986b5ee60e4b84d34a519a5edcd8c03e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 5 Jan 2016 21:27:54 +0200 Subject: Issue #20440: Cleaning up the code by using Py_SETREF. --- Modules/_datetimemodule.c | 24 ++++++------------------ Modules/_elementtree.c | 4 +--- Modules/_lsprof.c | 7 ++----- Modules/_pickle.c | 12 ++---------- Modules/readline.c | 4 +--- Objects/exceptions.c | 22 ++++++++-------------- Objects/frameobject.c | 6 +----- Objects/funcobject.c | 35 ++++++----------------------------- Objects/genobject.c | 12 ++---------- Objects/object.c | 8 +++----- Objects/typeobject.c | 6 ++---- Python/import.c | 10 +++------- Python/peephole.c | 4 +--- Python/sysmodule.c | 5 +---- 14 files changed, 39 insertions(+), 120 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3e5e195..4c8f3f6 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1057,10 +1057,8 @@ format_utcoffset(char *buf, size_t buflen, const char *sep, } /* Offset is normalized, so it is negative if days < 0 */ if (GET_TD_DAYS(offset) < 0) { - PyObject *temp = offset; sign = '-'; - offset = delta_negative((PyDateTime_Delta *)offset); - Py_DECREF(temp); + Py_SETREF(offset, delta_negative((PyDateTime_Delta *)offset)); if (offset == NULL) return -1; } @@ -3047,10 +3045,8 @@ tzinfo_fromutc(PyDateTime_TZInfo *self, PyObject *dt) if (dst == Py_None) goto Inconsistent; if (delta_bool((PyDateTime_Delta *)dst) != 0) { - PyObject *temp = result; - result = add_datetime_timedelta((PyDateTime_DateTime *)result, - (PyDateTime_Delta *)dst, 1); - Py_DECREF(temp); + Py_SETREF(result, add_datetime_timedelta((PyDateTime_DateTime *)result, + (PyDateTime_Delta *)dst, 1)); if (result == NULL) goto Fail; } @@ -4157,10 +4153,7 @@ datetime_datetime_now_impl(PyTypeObject *type, PyObject *tz) tz); if (self != NULL && tz != Py_None) { /* Convert UTC to tzinfo's zone. */ - PyObject *temp = self; - - self = _PyObject_CallMethodId(tz, &PyId_fromutc, "O", self); - Py_DECREF(temp); + self = _PyObject_CallMethodId(tz, &PyId_fromutc, "N", self); } return self; } @@ -4195,10 +4188,7 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) tzinfo); if (self != NULL && tzinfo != Py_None) { /* Convert UTC to tzinfo's zone. */ - PyObject *temp = self; - - self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "O", self); - Py_DECREF(temp); + self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "N", self); } return self; } @@ -4421,9 +4411,7 @@ datetime_subtract(PyObject *left, PyObject *right) return NULL; if (offdiff != NULL) { - PyObject *temp = result; - result = delta_subtract(result, offdiff); - Py_DECREF(temp); + Py_SETREF(result, delta_subtract(result, offdiff)); Py_DECREF(offdiff); } } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 580c53a..0effc3f 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -396,10 +396,8 @@ element_init(PyObject *self, PyObject *args, PyObject *kwds) Py_XDECREF(attrib); /* Replace the objects already pointed to by tag, text and tail. */ - tmp = self_elem->tag; Py_INCREF(tag); - self_elem->tag = tag; - Py_DECREF(tmp); + Py_SETREF(self_elem->tag, tag); tmp = self_elem->text; Py_INCREF(Py_None); diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 66e534f..d3ed758 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -762,7 +762,6 @@ profiler_dealloc(ProfilerObject *op) static int profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) { - PyObject *o; PyObject *timer = NULL; double timeunit = 0.0; int subcalls = 1; @@ -777,11 +776,9 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) if (setSubcalls(pObj, subcalls) < 0 || setBuiltins(pObj, builtins) < 0) return -1; - o = pObj->externalTimer; - pObj->externalTimer = timer; - Py_XINCREF(timer); - Py_XDECREF(o); pObj->externalTimerUnit = timeunit; + Py_XINCREF(timer); + Py_SETREF(pObj->externalTimer, timer); return 0; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b42f4f6..bb3330a 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4494,8 +4494,6 @@ Pickler_get_persid(PicklerObject *self) static int Pickler_set_persid(PicklerObject *self, PyObject *value) { - PyObject *tmp; - if (value == NULL) { PyErr_SetString(PyExc_TypeError, "attribute deletion is not supported"); @@ -4507,10 +4505,8 @@ Pickler_set_persid(PicklerObject *self, PyObject *value) return -1; } - tmp = self->pers_func; Py_INCREF(value); - self->pers_func = value; - Py_XDECREF(tmp); /* self->pers_func can be NULL, so be careful. */ + Py_SETREF(self->pers_func, value); return 0; } @@ -6946,8 +6942,6 @@ Unpickler_get_persload(UnpicklerObject *self) static int Unpickler_set_persload(UnpicklerObject *self, PyObject *value) { - PyObject *tmp; - if (value == NULL) { PyErr_SetString(PyExc_TypeError, "attribute deletion is not supported"); @@ -6960,10 +6954,8 @@ Unpickler_set_persload(UnpicklerObject *self, PyObject *value) return -1; } - tmp = self->pers_func; Py_INCREF(value); - self->pers_func = value; - Py_XDECREF(tmp); /* self->pers_func can be NULL, so be careful. */ + Py_SETREF(self->pers_func, value); return 0; } diff --git a/Modules/readline.c b/Modules/readline.c index 939ff1a..6930415 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -321,10 +321,8 @@ set_hook(const char *funcname, PyObject **hook_var, PyObject *args) Py_CLEAR(*hook_var); } else if (PyCallable_Check(function)) { - PyObject *tmp = *hook_var; Py_INCREF(function); - *hook_var = function; - Py_XDECREF(tmp); + Py_SETREF(*hook_var, function); } else { PyErr_Format(PyExc_TypeError, diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 85f9472..7374368 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -59,15 +59,11 @@ BaseException_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static int BaseException_init(PyBaseExceptionObject *self, PyObject *args, PyObject *kwds) { - PyObject *tmp; - if (!_PyArg_NoKeywords(Py_TYPE(self)->tp_name, kwds)) return -1; - tmp = self->args; - self->args = args; - Py_INCREF(self->args); - Py_XDECREF(tmp); + Py_INCREF(args); + Py_SETREF(self->args, args); return 0; } @@ -328,11 +324,10 @@ PyException_GetCause(PyObject *self) { /* Steals a reference to cause */ void -PyException_SetCause(PyObject *self, PyObject *cause) { - PyObject *old_cause = ((PyBaseExceptionObject *)self)->cause; - ((PyBaseExceptionObject *)self)->cause = cause; +PyException_SetCause(PyObject *self, PyObject *cause) +{ ((PyBaseExceptionObject *)self)->suppress_context = 1; - Py_XDECREF(old_cause); + Py_SETREF(((PyBaseExceptionObject *)self)->cause, cause); } PyObject * @@ -344,10 +339,9 @@ PyException_GetContext(PyObject *self) { /* Steals a reference to context */ void -PyException_SetContext(PyObject *self, PyObject *context) { - PyObject *old_context = ((PyBaseExceptionObject *)self)->context; - ((PyBaseExceptionObject *)self)->context = context; - Py_XDECREF(old_context); +PyException_SetContext(PyObject *self, PyObject *context) +{ + Py_SETREF(((PyBaseExceptionObject *)self)->context, context); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 37e626d..425a9ee 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -349,15 +349,11 @@ frame_gettrace(PyFrameObject *f, void *closure) static int frame_settrace(PyFrameObject *f, PyObject* v, void *closure) { - PyObject* old_value; - /* We rely on f_lineno being accurate when f_trace is set. */ f->f_lineno = PyFrame_GetLineNumber(f); - old_value = f->f_trace; Py_XINCREF(v); - f->f_trace = v; - Py_XDECREF(old_value); + Py_SETREF(f->f_trace, v); return 0; } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 13daaba..2967634 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -249,7 +249,6 @@ func_get_code(PyFunctionObject *op) static int func_set_code(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; Py_ssize_t nfree, nclosure; /* Not legal to del f.func_code or to set it to anything @@ -270,10 +269,8 @@ func_set_code(PyFunctionObject *op, PyObject *value) nclosure, nfree); return -1; } - tmp = op->func_code; Py_INCREF(value); - op->func_code = value; - Py_DECREF(tmp); + Py_SETREF(op->func_code, value); return 0; } @@ -287,8 +284,6 @@ func_get_name(PyFunctionObject *op) static int func_set_name(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del f.func_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -296,10 +291,8 @@ func_set_name(PyFunctionObject *op, PyObject *value) "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_DECREF(tmp); + Py_SETREF(op->func_name, value); return 0; } @@ -313,8 +306,6 @@ func_get_qualname(PyFunctionObject *op) static int func_set_qualname(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del f.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -322,10 +313,8 @@ func_set_qualname(PyFunctionObject *op, PyObject *value) "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_DECREF(tmp); + Py_SETREF(op->func_qualname, value); return 0; } @@ -343,8 +332,6 @@ func_get_defaults(PyFunctionObject *op) static int func_set_defaults(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - /* Legal to del f.func_defaults. * Can only set func_defaults to NULL or a tuple. */ if (value == Py_None) @@ -354,10 +341,8 @@ func_set_defaults(PyFunctionObject *op, PyObject *value) "__defaults__ must be set to a tuple object"); return -1; } - tmp = op->func_defaults; Py_XINCREF(value); - op->func_defaults = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_defaults, value); return 0; } @@ -375,8 +360,6 @@ func_get_kwdefaults(PyFunctionObject *op) static int func_set_kwdefaults(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - if (value == Py_None) value = NULL; /* Legal to del f.func_kwdefaults. @@ -386,10 +369,8 @@ func_set_kwdefaults(PyFunctionObject *op, PyObject *value) "__kwdefaults__ must be set to a dict object"); return -1; } - tmp = op->func_kwdefaults; Py_XINCREF(value); - op->func_kwdefaults = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_kwdefaults, value); return 0; } @@ -408,8 +389,6 @@ func_get_annotations(PyFunctionObject *op) static int func_set_annotations(PyFunctionObject *op, PyObject *value) { - PyObject *tmp; - if (value == Py_None) value = NULL; /* Legal to del f.func_annotations. @@ -420,10 +399,8 @@ func_set_annotations(PyFunctionObject *op, PyObject *value) "__annotations__ must be set to a dict object"); return -1; } - tmp = op->func_annotations; Py_XINCREF(value); - op->func_annotations = value; - Py_XDECREF(tmp); + Py_SETREF(op->func_annotations, value); return 0; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 00ebbf1..81a80b7 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -510,8 +510,6 @@ gen_get_name(PyGenObject *op) static int gen_set_name(PyGenObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del gen.gi_name or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -519,10 +517,8 @@ gen_set_name(PyGenObject *op, PyObject *value) "__name__ must be set to a string object"); return -1; } - tmp = op->gi_name; Py_INCREF(value); - op->gi_name = value; - Py_DECREF(tmp); + Py_SETREF(op->gi_name, value); return 0; } @@ -536,8 +532,6 @@ gen_get_qualname(PyGenObject *op) static int gen_set_qualname(PyGenObject *op, PyObject *value) { - PyObject *tmp; - /* Not legal to del gen.__qualname__ or to set it to anything * other than a string object. */ if (value == NULL || !PyUnicode_Check(value)) { @@ -545,10 +539,8 @@ gen_set_qualname(PyGenObject *op, PyObject *value) "__qualname__ must be set to a string object"); return -1; } - tmp = op->gi_qualname; Py_INCREF(value); - op->gi_qualname = value; - Py_DECREF(tmp); + Py_SETREF(op->gi_qualname, value); return 0; } diff --git a/Objects/object.c b/Objects/object.c index 417a97d..8072dbc 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1203,7 +1203,7 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { - PyObject *dict, **dictptr = _PyObject_GetDictPtr(obj); + PyObject **dictptr = _PyObject_GetDictPtr(obj); if (dictptr == NULL) { PyErr_SetString(PyExc_AttributeError, "This object has no __dict__"); @@ -1219,10 +1219,8 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - dict = *dictptr; - Py_XINCREF(value); - *dictptr = value; - Py_XDECREF(dict); + Py_INCREF(value); + Py_SETREF(*dictptr, value); return 0; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index c62255c..db15cf6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2092,7 +2092,7 @@ subtype_dict(PyObject *obj, void *context) static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject *dict, **dictptr; + PyObject **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -2123,10 +2123,8 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } - dict = *dictptr; Py_XINCREF(value); - *dictptr = value; - Py_XDECREF(dict); + Py_SETREF(*dictptr, value); return 0; } diff --git a/Python/import.c b/Python/import.c index 8d7bfe9..325b936 100644 --- a/Python/import.c +++ b/Python/import.c @@ -879,10 +879,8 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) if (PyUnicode_Compare(co->co_filename, oldname)) return; - tmp = co->co_filename; - co->co_filename = newname; - Py_INCREF(co->co_filename); - Py_DECREF(tmp); + Py_INCREF(newname); + Py_SETREF(co->co_filename, newname); constants = co->co_consts; n = PyTuple_GET_SIZE(constants); @@ -1327,10 +1325,8 @@ remove_importlib_frames(void) (always_trim || PyUnicode_CompareWithASCIIString(code->co_name, remove_frames) == 0)) { - PyObject *tmp = *outer_link; - *outer_link = next; Py_XINCREF(next); - Py_DECREF(tmp); + Py_SETREF(*outer_link, next); prev_link = outer_link; } else { diff --git a/Python/peephole.c b/Python/peephole.c index 59ad3b7..4bf786e 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -118,9 +118,7 @@ tuple_of_constants(unsigned char *codestr, Py_ssize_t n, /* If it's a BUILD_SET, use the PyTuple we just built to create a PyFrozenSet, and use that as the constant instead: */ if (codestr[0] == BUILD_SET) { - PyObject *tuple = newconst; - newconst = PyFrozenSet_New(tuple); - Py_DECREF(tuple); + Py_SETREF(newconst, PyFrozenSet_New(newconst)); if (newconst == NULL) return 0; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f784f75..53dd4a1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -436,10 +436,7 @@ trace_trampoline(PyObject *self, PyFrameObject *frame, return -1; } if (result != Py_None) { - PyObject *temp = frame->f_trace; - frame->f_trace = NULL; - Py_XDECREF(temp); - frame->f_trace = result; + Py_SETREF(frame->f_trace, result); } else { Py_DECREF(result); -- cgit v0.12 From a85e927e39b0ef3637c0bb6efa851494b8d92662 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Jan 2016 14:33:09 -0800 Subject: Issue #25802: Add an examples section to importlib. Thanks to Berker Peksag for the patch review. --- Doc/library/imp.rst | 15 +++++--- Doc/library/importlib.rst | 91 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index 68a6b68..420031a 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -81,7 +81,9 @@ This module provides an interface to the mechanisms used to implement the .. deprecated:: 3.3 Use :func:`importlib.util.find_spec` instead unless Python 3.3 compatibility is required, in which case use - :func:`importlib.find_loader`. + :func:`importlib.find_loader`. For example usage of the former case, + see the :ref:`importlib-examples` section of the :mod:`importlib` + documentation. .. function:: load_module(name, file, pathname, description) @@ -108,9 +110,12 @@ This module provides an interface to the mechanisms used to implement the If previously used in conjunction with :func:`imp.find_module` then consider using :func:`importlib.import_module`, otherwise use the loader returned by the replacement you chose for :func:`imp.find_module`. If you - called :func:`imp.load_module` and related functions directly then use the - classes in :mod:`importlib.machinery`, e.g. - ``importlib.machinery.SourceFileLoader(name, path).load_module()``. + called :func:`imp.load_module` and related functions directly with file + path arguments then use a combination of + :func:`importlib.util.spec_from_file_location` and + :func:`importlib.util.module_from_spec`. See the :ref:`importlib-examples` + section of the :mod:`importlib` documentation for details of the various + approaches. .. function:: new_module(name) @@ -119,7 +124,7 @@ This module provides an interface to the mechanisms used to implement the in ``sys.modules``. .. deprecated:: 3.4 - Use :class:`types.ModuleType` instead. + Use :func:`importlib.util.module_from_spec` instead. .. function:: reload(module) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index d800835..45766a7 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -256,7 +256,7 @@ ABC hierarchy:: module and *path* will be the value of :attr:`__path__` from the parent package. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may - use to make a more educated about what spec to return. + use to make a more educated guess about what spec to return. .. versionadded:: 3.4 @@ -306,7 +306,7 @@ ABC hierarchy:: within the :term:`path entry` to which it is assigned. If a spec cannot be found, ``None`` is returned. When passed in, ``target`` is a module object that the finder may use to make a more educated - about what spec to return. + guess about what spec to return. .. versionadded:: 3.4 @@ -1307,3 +1307,90 @@ an :term:`importer`. loader = importlib.machinery.SourceFileLoader lazy_loader = importlib.util.LazyLoader.factory(loader) finder = importlib.machinery.FileFinder(path, [(lazy_loader, suffixes)]) + +.. _importlib-examples: + +Examples +-------- + +To programmatically import a module, use :func:`importlib.import_module`. +:: + + import importlib + + itertools = importlib.import_module('itertools') + +If you need to find out if a module can be imported without actually doing the +import, then you should use :func:`importlib.util.find_spec`. +:: + + import importlib.util + import sys + + # For illustrative purposes. + name = 'itertools' + + spec = importlib.util.find_spec(name) + if spec is None: + print("can't find the itertools module") + else: + # If you chose to perform the actual import. + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Adding the module to sys.modules is optional. + sys.modules[name] = module + +To import a Python source file directly, use the following recipe +(Python 3.4 and newer only):: + + import importlib.util + import sys + + # For illustrative purposes. + import tokenize + file_path = tokenize.__file__ + module_name = tokenize.__name__ + + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + # Optional; only necessary if you want to be able to import the module + # by name later. + sys.modules[module_name] = module + +Import itself is implemented in Python code, making it possible to +expose most of the import machinery through importlib. The following +helps illustrate the various APIs that importlib exposes by providing an +approximate implementation of +:func:`importlib.import_module` (Python 3.4 and newer for importlib usage, +Python 3.6 and newer for other parts of the code). +:: + + import importlib.util + import sys + + def import_module(name, package=None): + """An approximate implementation of import.""" + absolute_name = importlib.util.resolve_name(name, package) + try: + return sys.modules[absolute_name] + except KeyError: + pass + + path = None + if '.' in absolute_name: + parent_name, _, child_name = absolute_name.rpartition('.') + parent_module = import_module(parent_name) + path = parent_module.spec.submodule_search_locations + for finder in sys.meta_path: + spec = finder.find_spec(absolute_name, path) + if spec is not None: + break + else: + raise ImportError(f'No module named {absolute_name!r}') + module = spec.loader.create_module(spec) + spec.loader.exec_module(module) + sys.modules[absolute_name] = module + if path is not None: + setattr(parent_module, child_name, module) + return module -- cgit v0.12 From 32d1e56bda420602fd4106dbc51ea1dc6f02ba14 Mon Sep 17 00:00:00 2001 From: Mark Hammond Date: Mon, 11 Jan 2016 14:53:01 +1100 Subject: Issue #26070: py.exe launcher fails to find in-place built binaries from earlier Python versions. --- PC/launcher.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PC/launcher.c b/PC/launcher.c index b379a38..e6baae5 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -171,6 +171,9 @@ static wchar_t * location_checks[] = { L"\\", L"\\PCBuild\\win32\\", L"\\PCBuild\\amd64\\", + // To support early 32bit versions of Python that stuck the build binaries + // directly in PCBuild... + L"\\PCBuild\\", NULL }; -- cgit v0.12 From 96b531abd5bac9707e1b8f714c4d9259958726f9 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Mon, 11 Jan 2016 07:09:42 -0800 Subject: Issue #26069: Remove the deprecated apis in the trace module. --- Lib/test/test_trace.py | 48 +----------------------------------------------- Lib/trace.py | 43 ------------------------------------------- Misc/NEWS | 2 ++ 3 files changed, 3 insertions(+), 90 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 03dff84..55a3bfa 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,10 @@ import os -import io import sys from test.support import TESTFN, rmtree, unlink, captured_stdout import unittest import trace -from trace import CoverageResults, Trace +from trace import Trace from test.tracedmodules import testmod @@ -366,50 +365,5 @@ class Test_Ignore(unittest.TestCase): self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) -class TestDeprecatedMethods(unittest.TestCase): - - def test_deprecated_usage(self): - sio = io.StringIO() - with self.assertWarns(DeprecationWarning): - trace.usage(sio) - self.assertIn('Usage:', sio.getvalue()) - - def test_deprecated_Ignore(self): - with self.assertWarns(DeprecationWarning): - trace.Ignore() - - def test_deprecated_modname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.modname("spam")) - - def test_deprecated_fullmodname(self): - with self.assertWarns(DeprecationWarning): - self.assertEqual("spam", trace.fullmodname("spam")) - - def test_deprecated_find_lines_from_code(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines_from_code(foo.__code__, ["eggs"]) - - def test_deprecated_find_lines(self): - with self.assertWarns(DeprecationWarning): - def foo(): - pass - trace.find_lines(foo.__code__, ["eggs"]) - - def test_deprecated_find_strings(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_strings(fd.name) - - def test_deprecated_find_executable_linenos(self): - with open(TESTFN, 'w') as fd: - self.addCleanup(unlink, TESTFN) - with self.assertWarns(DeprecationWarning): - trace.find_executable_linenos(fd.name) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index f108266..b768829 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -58,7 +58,6 @@ import inspect import gc import dis import pickle -from warnings import warn as _warn from time import monotonic as _time try: @@ -810,47 +809,5 @@ def main(argv=None): if not no_report: results.write_results(missing, summary=summary, coverdir=coverdir) -# Deprecated API -def usage(outfile): - _warn("The trace.usage() function is deprecated", - DeprecationWarning, 2) - _usage(outfile) - -class Ignore(_Ignore): - def __init__(self, modules=None, dirs=None): - _warn("The class trace.Ignore is deprecated", - DeprecationWarning, 2) - _Ignore.__init__(self, modules, dirs) - -def modname(path): - _warn("The trace.modname() function is deprecated", - DeprecationWarning, 2) - return _modname(path) - -def fullmodname(path): - _warn("The trace.fullmodname() function is deprecated", - DeprecationWarning, 2) - return _fullmodname(path) - -def find_lines_from_code(code, strs): - _warn("The trace.find_lines_from_code() function is deprecated", - DeprecationWarning, 2) - return _find_lines_from_code(code, strs) - -def find_lines(code, strs): - _warn("The trace.find_lines() function is deprecated", - DeprecationWarning, 2) - return _find_lines(code, strs) - -def find_strings(filename, encoding=None): - _warn("The trace.find_strings() function is deprecated", - DeprecationWarning, 2) - return _find_strings(filename, encoding=None) - -def find_executable_linenos(filename): - _warn("The trace.find_executable_linenos() function is deprecated", - DeprecationWarning, 2) - return _find_executable_linenos(filename) - if __name__=='__main__': main() diff --git a/Misc/NEWS b/Misc/NEWS index 60799b6..d54c489 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,8 @@ Core and Builtins Library ------- +- Issue #26069: Remove the deprecated apis in the trace module. + - Issue #22138: Fix mock.patch behavior when patching descriptors. Restore original values after patching. Patch contributed by Sean McCully. -- cgit v0.12 From 37dc2b28834c95d24746c2958e9ea31d3e8d1968 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 11 Jan 2016 15:15:01 -0500 Subject: Issue #25486: Resurrect inspect.getargspec in 3.6. Backout a565aad5d6e1. The decision is that we shouldn't remove popular APIs (however long they are depreacted) from Python 3, while 2.7 is still around and supported. --- Doc/library/inspect.rst | 18 ++++++++++++++++++ Doc/whatsnew/3.6.rst | 3 --- Lib/inspect.py | 25 +++++++++++++++++++++++++ Lib/test/test_inspect.py | 40 +++++++++++++++++++++++++++++++++++----- Misc/NEWS | 3 +-- 5 files changed, 79 insertions(+), 10 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8045d85..8e8d725 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -796,6 +796,24 @@ Classes and functions classes using multiple inheritance and their descendants will appear multiple times. + +.. function:: getargspec(func) + + Get the names and default values of a Python function's arguments. A + :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is + returned. *args* is a list of the argument names. *varargs* and *keywords* + are the names of the ``*`` and ``**`` arguments or ``None``. *defaults* is a + tuple of default argument values or ``None`` if there are no default + arguments; if this tuple has *n* elements, they correspond to the last + *n* elements listed in *args*. + + .. deprecated:: 3.0 + Use :func:`signature` and + :ref:`Signature Object `, which provide a + better introspecting API for callables. This function will be removed + in Python 3.6. + + .. function:: getfullargspec(func) Get the names and default values of a Python function's arguments. A diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bf5161d..f7dbbee 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -218,9 +218,6 @@ Removed API and Feature Removals ------------------------ -* ``inspect.getargspec()`` was removed (was deprecated since CPython 3.0). - :func:`inspect.getfullargspec` is an almost drop in replacement. - * ``inspect.getmoduleinfo()`` was removed (was deprecated since CPython 3.3). :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. diff --git a/Lib/inspect.py b/Lib/inspect.py index 7615e52..74f3b3c 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1004,6 +1004,31 @@ def _getfullargs(co): varkw = co.co_varnames[nargs] return args, varargs, kwonlyargs, varkw + +ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') + +def getargspec(func): + """Get the names and default values of a function's arguments. + + A tuple of four things is returned: (args, varargs, keywords, defaults). + 'args' is a list of the argument names, including keyword-only argument names. + 'varargs' and 'keywords' are the names of the * and ** arguments or None. + 'defaults' is an n-tuple of the default values of the last n arguments. + + Use the getfullargspec() API for Python 3 code, as annotations + and keyword arguments are supported. getargspec() will raise ValueError + if the func has either annotations or keyword arguments. + """ + warnings.warn("inspect.getargspec() is deprecated, " + "use inspect.signature() instead", DeprecationWarning, + stacklevel=2) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ + getfullargspec(func) + if kwonlyargs or ann: + raise ValueError("Function has keyword-only arguments or annotations" + ", use getfullargspec() API which can support them") + return ArgSpec(args, varargs, varkw, defaults) + FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index a88e7fd..283c922 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -628,6 +628,18 @@ class TestClassesAndFunctions(unittest.TestCase): got = inspect.getmro(D) self.assertEqual(expected, got) + def assertArgSpecEquals(self, routine, args_e, varargs_e=None, + varkw_e=None, defaults_e=None, formatted=None): + with self.assertWarns(DeprecationWarning): + args, varargs, varkw, defaults = inspect.getargspec(routine) + self.assertEqual(args, args_e) + self.assertEqual(varargs, varargs_e) + self.assertEqual(varkw, varkw_e) + self.assertEqual(defaults, defaults_e) + if formatted is not None: + self.assertEqual(inspect.formatargspec(args, varargs, varkw, defaults), + formatted) + def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, kwonlyargs_e=[], kwonlydefaults_e=None, @@ -646,6 +658,23 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs, kwonlydefaults, ann), formatted) + def test_getargspec(self): + self.assertArgSpecEquals(mod.eggs, ['x', 'y'], formatted='(x, y)') + + self.assertArgSpecEquals(mod.spam, + ['a', 'b', 'c', 'd', 'e', 'f'], + 'g', 'h', (3, 4, 5), + '(a, b, c, d=3, e=4, f=5, *g, **h)') + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyworded, []) + + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.annotated, []) + self.assertRaises(ValueError, self.assertArgSpecEquals, + mod2.keyword_only_arg, []) + + def test_getfullargspec(self): self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', kwonlyargs_e=['arg2'], @@ -659,19 +688,20 @@ class TestClassesAndFunctions(unittest.TestCase): kwonlyargs_e=['arg'], formatted='(*, arg)') - def test_fullargspec_api_ignores_wrapped(self): + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) def ham(x, y): pass # Basic check + self.assertArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(ham, ['x', 'y'], formatted='(x, y)') self.assertFullArgSpecEquals(functools.partial(ham), ['x', 'y'], formatted='(x, y)') # Other variants def check_method(f): - self.assertFullArgSpecEquals(f, ['self', 'x', 'y'], - formatted='(self, x, y)') + self.assertArgSpecEquals(f, ['self', 'x', 'y'], + formatted='(self, x, y)') class C: @functools.wraps(mod.spam) def ham(self, x, y): @@ -749,11 +779,11 @@ class TestClassesAndFunctions(unittest.TestCase): with self.assertRaises(TypeError): inspect.getfullargspec(builtin) - def test_getfullargspec_method(self): + def test_getargspec_method(self): class A(object): def m(self): pass - self.assertFullArgSpecEquals(A.m, ['self']) + self.assertArgSpecEquals(A.m, ['self']) def test_classify_newstyle(self): class A(object): diff --git a/Misc/NEWS b/Misc/NEWS index 593ed24..089388e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -429,8 +429,7 @@ Library - Issue #23661: unittest.mock side_effects can now be exceptions again. This was a regression vs Python 3.4. Patch from Ignacio Rossi -- Issue #13248: Remove deprecated inspect.getargspec and inspect.getmoduleinfo - functions. +- Issue #13248: Remove deprecated inspect.getmoduleinfo function. - Issue #25578: Fix (another) memory leak in SSLSocket.getpeercer(). -- cgit v0.12 From 121edbf7e26ad0ac887ac4a2bf46316deb05737e Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Tue, 12 Jan 2016 06:18:32 -0800 Subject: Issue25347 - Format the error message output of mock's assert_has_calls method. Patch contributed by Robert Zimmerman. --- Lib/unittest/mock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 976f663..21f49fa 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -820,7 +820,7 @@ class NonCallableMock(Base): if expected not in all_calls: raise AssertionError( 'Calls not found.\nExpected: %r\n' - 'Actual: %r' % (calls, self.mock_calls) + 'Actual: %r' % (_CallList(calls), self.mock_calls) ) from cause return -- cgit v0.12 From 436831dbe47e66afb0a4f5927859073015dbd4bb Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 13 Jan 2016 07:46:54 -0800 Subject: Issue22642 - Convert trace module's option handling mechanism from getopt to argparse. Patch contributed by SilentGhost. --- Lib/test/test_trace.py | 22 ++++ Lib/trace.py | 339 +++++++++++++++++++------------------------------ 2 files changed, 156 insertions(+), 205 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 55a3bfa..1d894aa 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,6 +1,7 @@ import os import sys from test.support import TESTFN, rmtree, unlink, captured_stdout +from test.support.script_helper import assert_python_ok, assert_python_failure import unittest import trace @@ -364,6 +365,27 @@ class Test_Ignore(unittest.TestCase): # Matched before. self.assertTrue(ignore.names(jn('bar', 'baz.py'), 'baz')) +class TestCommandLine(unittest.TestCase): + + def test_failures(self): + _errors = ( + (b'filename is missing: required with the main options', '-l', '-T'), + (b'cannot specify both --listfuncs and (--trace or --count)', '-lc'), + (b'argument -R/--no-report: not allowed with argument -r/--report', '-rR'), + (b'must specify one of --trace, --count, --report, --listfuncs, or --trackcalls', '-g'), + (b'-r/--report requires -f/--file', '-r'), + (b'--summary can only be used with --count or --report', '-sT'), + (b'unrecognized arguments: -y', '-y')) + for message, *args in _errors: + *_, stderr = assert_python_failure('-m', 'trace', *args) + self.assertIn(message, stderr) + + def test_listfuncs_flag_success(self): + with open(TESTFN, 'w') as fd: + self.addCleanup(unlink, TESTFN) + fd.write("a = 1\n") + status, stdout, stderr = assert_python_ok('-m', 'trace', '-l', TESTFN) + self.assertIn(b'functions called:', stdout) if __name__ == '__main__': unittest.main() diff --git a/Lib/trace.py b/Lib/trace.py index b768829..7afbe76 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -48,6 +48,7 @@ Sample use, programmatically r.write_results(show_missing=True, coverdir="/tmp") """ __all__ = ['Trace', 'CoverageResults'] +import argparse import linecache import os import re @@ -76,51 +77,6 @@ else: sys.settrace(None) threading.settrace(None) -def _usage(outfile): - outfile.write("""Usage: %s [OPTIONS] [ARGS] - -Meta-options: ---help Display this help then exit. ---version Output version information then exit. - -Otherwise, exactly one of the following three options must be given: --t, --trace Print each line to sys.stdout before it is executed. --c, --count Count the number of times each line is executed - and write the counts to .cover for each - module executed, in the module's directory. - See also `--coverdir', `--file', `--no-report' below. --l, --listfuncs Keep track of which functions are executed at least - once and write the results to sys.stdout after the - program exits. --T, --trackcalls Keep track of caller/called pairs and write the - results to sys.stdout after the program exits. --r, --report Generate a report from a counts file; do not execute - any code. `--file' must specify the results file to - read, which must have been created in a previous run - with `--count --file=FILE'. - -Modifiers: --f, --file= File to accumulate counts over several runs. --R, --no-report Do not generate the coverage report files. - Useful if you want to accumulate over several runs. --C, --coverdir= Directory where the report files. The coverage - report for . is written to file - //.cover. --m, --missing Annotate executable lines that were not executed - with '>>>>>> '. --s, --summary Write a brief summary on stdout for each file. - (Can only be used with --count or --report.) --g, --timing Prefix each line with the time since the program started. - Only used while tracing. - -Filters, may be repeated multiple times: ---ignore-module= Ignore the given module(s) and its submodules - (if it is a package). Accepts comma separated - list of module names ---ignore-dir= Ignore files in the given directory (multiple - directories can be joined by os.pathsep). -""" % sys.argv[0]) - PRAGMA_NOCOVER = "#pragma NO COVER" # Simple rx to find lines with no code. @@ -264,7 +220,13 @@ class CoverageResults: def write_results(self, show_missing=True, summary=False, coverdir=None): """ - @param coverdir + Write the coverage results. + + :param show_missing: Show lines that had no hits. + :param summary: Include coverage summary per module. + :param coverdir: If None, the results of each module are placed in it's + directory, otherwise it is included in the directory + specified. """ if self.calledfuncs: print() @@ -646,168 +608,135 @@ class Trace: calledfuncs=self._calledfuncs, callers=self._callers) -def _err_exit(msg): - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.exit(1) - -def main(argv=None): - import getopt - - if argv is None: - argv = sys.argv +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version='trace 2.0') + + grp = parser.add_argument_group('Main options', + 'One of these (or --report) must be given') + + grp.add_argument('-c', '--count', action='store_true', + help='Count the number of times each line is executed and write ' + 'the counts to .cover for each module executed, in ' + 'the module\'s directory. See also --coverdir, --file, ' + '--no-report below.') + grp.add_argument('-t', '--trace', action='store_true', + help='Print each line to sys.stdout before it is executed') + grp.add_argument('-l', '--listfuncs', action='store_true', + help='Keep track of which functions are executed at least once ' + 'and write the results to sys.stdout after the program exits. ' + 'Cannot be specified alongside --trace or --count.') + grp.add_argument('-T', '--trackcalls', action='store_true', + help='Keep track of caller/called pairs and write the results to ' + 'sys.stdout after the program exits.') + + grp = parser.add_argument_group('Modifiers') + + _grp = grp.add_mutually_exclusive_group() + _grp.add_argument('-r', '--report', action='store_true', + help='Generate a report from a counts file; does not execute any ' + 'code. --file must specify the results file to read, which ' + 'must have been created in a previous run with --count ' + '--file=FILE') + _grp.add_argument('-R', '--no-report', action='store_true', + help='Do not generate the coverage report files. ' + 'Useful if you want to accumulate over several runs.') + + grp.add_argument('-f', '--file', + help='File to accumulate counts over several runs') + grp.add_argument('-C', '--coverdir', + help='Directory where the report files go. The coverage report ' + 'for . will be written to file ' + '//.cover') + grp.add_argument('-m', '--missing', action='store_true', + help='Annotate executable lines that were not executed with ' + '">>>>>> "') + grp.add_argument('-s', '--summary', action='store_true', + help='Write a brief summary for each file to sys.stdout. ' + 'Can only be used with --count or --report') + grp.add_argument('-g', '--timing', action='store_true', + help='Prefix each line with the time since the program started. ' + 'Only used while tracing') + + grp = parser.add_argument_group('Filters', + 'Can be specified multiple times') + grp.add_argument('--ignore-module', action='append', default=[], + help='Ignore the given module(s) and its submodules' + '(if it is a package). Accepts comma separated list of ' + 'module names.') + grp.add_argument('--ignore-dir', action='append', default=[], + help='Ignore files in the given directory ' + '(multiple directories can be joined by os.pathsep).') + + parser.add_argument('filename', nargs='?', + help='file to run as main program') + parser.add_argument('arguments', nargs=argparse.REMAINDER, + help='arguments to the program') + + opts = parser.parse_args() + + if opts.ignore_dir: + rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) + _prefix = os.path.join(sys.base_prefix, *rel_path) + _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + + def parse_ignore_dir(s): + s = os.path.expanduser(os.path.expandvars(s)) + s = s.replace('$prefix', _prefix).replace('$exec_prefix', _exec_prefix) + return os.path.normpath(s) + + opts.ignore_module = [mod.strip() + for i in opts.ignore_module for mod in i.split(',')] + opts.ignore_dir = [parse_ignore_dir(s) + for i in opts.ignore_dir for s in i.split(os.pathsep)] + + if opts.report: + if not opts.file: + parser.error('-r/--report requires -f/--file') + results = CoverageResults(infile=opts.file, outfile=opts.file) + return results.write_results(opts.missing, opts.summary, opts.coverdir) + + if not any([opts.trace, opts.count, opts.listfuncs, opts.trackcalls]): + parser.error('must specify one of --trace, --count, --report, ' + '--listfuncs, or --trackcalls') + + if opts.listfuncs and (opts.count or opts.trace): + parser.error('cannot specify both --listfuncs and (--trace or --count)') + + if opts.summary and not opts.count: + parser.error('--summary can only be used with --count or --report') + + if opts.filename is None: + parser.error('filename is missing: required with the main options') + + sys.argv = opts.filename, *opts.arguments + sys.path[0] = os.path.dirname(opts.filename) + + t = Trace(opts.count, opts.trace, countfuncs=opts.listfuncs, + countcallers=opts.trackcalls, ignoremods=opts.ignore_module, + ignoredirs=opts.ignore_dir, infile=opts.file, + outfile=opts.file, timing=opts.timing) try: - opts, prog_argv = getopt.getopt(argv[1:], "tcrRf:d:msC:lTg", - ["help", "version", "trace", "count", - "report", "no-report", "summary", - "file=", "missing", - "ignore-module=", "ignore-dir=", - "coverdir=", "listfuncs", - "trackcalls", "timing"]) - - except getopt.error as msg: - sys.stderr.write("%s: %s\n" % (sys.argv[0], msg)) - sys.stderr.write("Try `%s --help' for more information\n" - % sys.argv[0]) - sys.exit(1) - - trace = 0 - count = 0 - report = 0 - no_report = 0 - counts_file = None - missing = 0 - ignore_modules = [] - ignore_dirs = [] - coverdir = None - summary = 0 - listfuncs = False - countcallers = False - timing = False - - for opt, val in opts: - if opt == "--help": - _usage(sys.stdout) - sys.exit(0) - - if opt == "--version": - sys.stdout.write("trace 2.0\n") - sys.exit(0) - - if opt == "-T" or opt == "--trackcalls": - countcallers = True - continue - - if opt == "-l" or opt == "--listfuncs": - listfuncs = True - continue - - if opt == "-g" or opt == "--timing": - timing = True - continue - - if opt == "-t" or opt == "--trace": - trace = 1 - continue - - if opt == "-c" or opt == "--count": - count = 1 - continue - - if opt == "-r" or opt == "--report": - report = 1 - continue - - if opt == "-R" or opt == "--no-report": - no_report = 1 - continue - - if opt == "-f" or opt == "--file": - counts_file = val - continue - - if opt == "-m" or opt == "--missing": - missing = 1 - continue - - if opt == "-C" or opt == "--coverdir": - coverdir = val - continue - - if opt == "-s" or opt == "--summary": - summary = 1 - continue - - if opt == "--ignore-module": - for mod in val.split(","): - ignore_modules.append(mod.strip()) - continue - - if opt == "--ignore-dir": - for s in val.split(os.pathsep): - s = os.path.expandvars(s) - # should I also call expanduser? (after all, could use $HOME) - - s = s.replace("$prefix", - os.path.join(sys.base_prefix, "lib", - "python" + sys.version[:3])) - s = s.replace("$exec_prefix", - os.path.join(sys.base_exec_prefix, "lib", - "python" + sys.version[:3])) - s = os.path.normpath(s) - ignore_dirs.append(s) - continue - - assert 0, "Should never get here" - - if listfuncs and (count or trace): - _err_exit("cannot specify both --listfuncs and (--trace or --count)") - - if not (count or trace or report or listfuncs or countcallers): - _err_exit("must specify one of --trace, --count, --report, " - "--listfuncs, or --trackcalls") - - if report and no_report: - _err_exit("cannot specify both --report and --no-report") - - if report and not counts_file: - _err_exit("--report requires a --file") - - if no_report and len(prog_argv) == 0: - _err_exit("missing name of file to run") - - # everything is ready - if report: - results = CoverageResults(infile=counts_file, outfile=counts_file) - results.write_results(missing, summary=summary, coverdir=coverdir) - else: - sys.argv = prog_argv - progname = prog_argv[0] - sys.path[0] = os.path.split(progname)[0] - - t = Trace(count, trace, countfuncs=listfuncs, - countcallers=countcallers, ignoremods=ignore_modules, - ignoredirs=ignore_dirs, infile=counts_file, - outfile=counts_file, timing=timing) - try: - with open(progname) as fp: - code = compile(fp.read(), progname, 'exec') - # try to emulate __main__ namespace as much as possible - globs = { - '__file__': progname, - '__name__': '__main__', - '__package__': None, - '__cached__': None, - } - t.runctx(code, globs, globs) - except OSError as err: - _err_exit("Cannot run file %r because: %s" % (sys.argv[0], err)) - except SystemExit: - pass + with open(opts.filename) as fp: + code = compile(fp.read(), opts.filename, 'exec') + # try to emulate __main__ namespace as much as possible + globs = { + '__file__': opts.filename, + '__name__': '__main__', + '__package__': None, + '__cached__': None, + } + t.runctx(code, globs, globs) + except OSError as err: + sys.exit("Cannot run file %r because: %s" % (sys.argv[0], err)) + except SystemExit: + pass - results = t.results() + results = t.results() - if not no_report: - results.write_results(missing, summary=summary, coverdir=coverdir) + if not opts.no_report: + results.write_results(opts.missing, opts.summary, opts.coverdir) if __name__=='__main__': main() -- cgit v0.12 From 4e280a6f9fe9b052912de1aa11205346f91bb279 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Wed, 13 Jan 2016 07:48:57 -0800 Subject: Add a NEWS entry for Issue #22642. --- Misc/NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 54c70fc..08f057d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,9 @@ Core and Builtins Library ------- +- Issue #22642: Convert trace module option parsing mechanism to argparse. + Patch contributed by SilentGhost. + - Issue #24705: Fix sysconfig._parse_makefile not expanding ${} vars appearing before $() vars. -- cgit v0.12 From 86f7109dadea2c5c97c15e1878b862e769199d1d Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Thu, 14 Jan 2016 00:11:39 -0800 Subject: Issue #25822: Add docstrings to the fields of urllib.parse results. Patch contributed by Swati Jaiswal. --- Lib/urllib/parse.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++-- Misc/NEWS | 3 +++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 5e2155c..cecd0b9 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -224,8 +224,71 @@ class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): from collections import namedtuple _DefragResultBase = namedtuple('DefragResult', 'url fragment') -_SplitResultBase = namedtuple('SplitResult', 'scheme netloc path query fragment') -_ParseResultBase = namedtuple('ParseResult', 'scheme netloc path params query fragment') +_SplitResultBase = namedtuple( + 'SplitResult', 'scheme netloc path query fragment') +_ParseResultBase = namedtuple( + 'ParseResult', 'scheme netloc path params query fragment') + +_DefragResultBase.__doc__ = """ +DefragResult(url, fragment) + +A 2-tuple that contains the url without fragment identifier and the fragment +identifier as a separate argument. +""" + +_DefragResultBase.url.__doc__ = """The URL with no fragment identifier.""" + +_DefragResultBase.fragment.__doc__ = """ +Fragment identifier separated from URL, that allows indirect identification of a +secondary resource by reference to a primary resource and additional identifying +information. +""" + +_SplitResultBase.__doc__ = """ +SplitResult(scheme, netloc, path, query, fragment) + +A 5-tuple that contains the different components of a URL. Similar to +ParseResult, but does not split params. +""" + +_SplitResultBase.scheme.__doc__ = """Specifies URL scheme for the request.""" + +_SplitResultBase.netloc.__doc__ = """ +Network location where the request is made to. +""" + +_SplitResultBase.path.__doc__ = """ +The hierarchical path, such as the path to a file to download. +""" + +_SplitResultBase.query.__doc__ = """ +The query component, that contains non-hierarchical data, that along with data +in path component, identifies a resource in the scope of URI's scheme and +network location. +""" + +_SplitResultBase.fragment.__doc__ = """ +Fragment identifier, that allows indirect identification of a secondary resource +by reference to a primary resource and additional identifying information. +""" + +_ParseResultBase.__doc__ = """ +ParseResult(scheme, netloc, path, params, query, fragment) + +A 6-tuple that contains components of a parsed URL. +""" + +_ParseResultBase.scheme.__doc__ = _SplitResultBase.scheme.__doc__ +_ParseResultBase.netloc.__doc__ = _SplitResultBase.netloc.__doc__ +_ParseResultBase.path.__doc__ = _SplitResultBase.path.__doc__ +_ParseResultBase.params.__doc__ = """ +Parameters for last path element used to dereference the URI in order to provide +access to perform some operation on the resource. +""" + +_ParseResultBase.query.__doc__ = _SplitResultBase.query.__doc__ +_ParseResultBase.fragment.__doc__ = _SplitResultBase.fragment.__doc__ + # For backwards compatibility, alias _NetlocResultMixinStr # ResultBase is no longer part of the documented API, but it is diff --git a/Misc/NEWS b/Misc/NEWS index 08f057d..c3a2574 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -128,6 +128,9 @@ Core and Builtins Library ------- +- Issue #25822: Add docstrings to the fields of urllib.parse results. + Patch contributed by Swati Jaiswal. + - Issue #22642: Convert trace module option parsing mechanism to argparse. Patch contributed by SilentGhost. -- cgit v0.12 From 8df98483222d480db087860656d3e369f2338663 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 14 Jan 2016 13:26:43 +0000 Subject: Issue #25940: test_ssl is working again --- Lib/test/test_ssl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index b3aee62..f95cfba 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1343,7 +1343,6 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -@unittest.skipIf(True, "temporarily disabled: see #25940") class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1712,7 +1711,6 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - @unittest.skipIf(True, "temporarily disabled: see #25940") def test_handshake(self): with support.transient_internet(REMOTE_HOST): sock = socket.socket(socket.AF_INET) -- cgit v0.12 From 63b8505281f9c26f7304b3e00d658b429b862d5b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 15 Jan 2016 13:33:03 -0800 Subject: Issue #25791: Raise an ImportWarning when __spec__ or __package__ are not defined for a relative import. This is the start of work to try and clean up import semantics to rely more on a module's spec than on the myriad attributes that get set on a module. Thanks to Rose Ames for the patch. --- Doc/reference/import.rst | 3 +- Doc/whatsnew/3.6.rst | 11 +- Lib/importlib/_bootstrap.py | 6 + .../test_importlib/import_/test___package__.py | 40 +- Misc/NEWS | 3 + Python/import.c | 64 ++- Python/importlib.h | 559 +++++++++++---------- 7 files changed, 376 insertions(+), 310 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 2144c1f..a162851 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -554,7 +554,8 @@ the module. details. This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. + relative imports for main modules -- as defined in :pep:`366` -- + when ``__spec__`` is not defined. .. attribute:: __spec__ diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f7dbbee..a5b9be2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -209,7 +209,12 @@ Deprecated features * The ``pyvenv`` script has been deprecated in favour of ``python3 -m venv``. This prevents confusion as to what Python interpreter ``pyvenv`` is connected to and thus what Python interpreter will be used by the virtual - environment. See :issue:`25154`. + environment. (Contributed by Brett Cannon in :issue:`25154`.) + +* When performing a relative import, falling back on ``__name__`` and + ``__path__`` from the calling module when ``__spec__`` or + ``__package__`` are not defined now raises an :exc:`ImportWarning`. + (Contributed by Rose Ames in :issue:`25791`.) Removed @@ -251,6 +256,10 @@ Changes in the Python API :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. +* When performing a relative import, ``__spec__.parent`` is used + is ``__spec__`` is defined instead of ``__package__``. + (Contributed by Rose Ames in :issue:`25791`.) + Changes in the C API -------------------- diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6f62bb3..9adcf7b 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,8 +1032,14 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + spec = globals.get('__spec__') + if spec is not None: + return spec.parent package = globals.get('__package__') if package is None: + _warnings.warn("can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, stacklevel=3) package = globals['__name__'] if '__path__' not in globals: package = package.rpartition('.')[0] diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index c7d3a2a..08e41c9 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -33,31 +33,39 @@ class Using__package__: """ - def test_using___package__(self): - # [__package__] + def import_module(self, globals_): with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals={'__package__': 'pkg.fake'}, + globals=globals_, fromlist=['attr'], level=2) + return module + + def test_using___package__(self): + # [__package__] + module = self.import_module({'__package__': 'pkg.fake'}) self.assertEqual(module.__name__, 'pkg') - def test_using___name__(self, package_as_None=False): + def test_using___name__(self): # [__name__] - globals_ = {'__name__': 'pkg.fake', '__path__': []} - if package_as_None: - globals_['__package__'] = None - with self.mock_modules('pkg.__init__', 'pkg.fake') as importer: - with util.import_state(meta_path=[importer]): - self.__import__('pkg.fake') - module = self.__import__('', globals= globals_, - fromlist=['attr'], level=2) - self.assertEqual(module.__name__, 'pkg') + module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_using___name__(self): + with self.assertWarns(ImportWarning): + self.import_module({'__name__': 'pkg.fake', '__path__': []}) def test_None_as___package__(self): # [None] - self.test_using___name__(package_as_None=True) + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + self.assertEqual(module.__name__, 'pkg') + + def test_prefers___spec__(self): + globals = {'__spec__': FakeSpec()} + with self.assertRaises(SystemError): + self.__import__('', globals, {}, ['relimport'], 1) def test_bad__package__(self): globals = {'__package__': ''} @@ -70,6 +78,10 @@ class Using__package__: self.__import__('', globals, {}, ['relimport'], 1) +class FakeSpec: + parent = '' + + class Using__package__PEP302(Using__package__): mock_modules = util.mock_modules diff --git a/Misc/NEWS b/Misc/NEWS index da53b95..f87703f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #25791: Trying to resolve a relative import without __spec__ or + __package__ defined now raises an ImportWarning + - Issue #25961: Disallowed null characters in the type name. - Issue #25973: Fix segfault when an invalid nonlocal statement binds a name diff --git a/Python/import.c b/Python/import.c index 325b936..c1071c0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1359,6 +1359,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *final_mod = NULL; PyObject *mod = NULL; PyObject *package = NULL; + PyObject *spec = NULL; PyObject *globals = NULL; PyObject *fromlist = NULL; PyInterpreterState *interp = PyThreadState_GET()->interp; @@ -1414,39 +1415,60 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error; } else if (level > 0) { - package = _PyDict_GetItemId(globals, &PyId___package__); + spec = _PyDict_GetItemId(globals, &PyId___spec__); + if (spec != NULL) { + package = PyObject_GetAttrString(spec, "parent"); + } if (package != NULL && package != Py_None) { - Py_INCREF(package); if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "package must be a string"); + PyErr_SetString(PyExc_TypeError, + "__spec__.parent must be a string"); goto error; } } else { - package = _PyDict_GetItemId(globals, &PyId___name__); - if (package == NULL) { - PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); - goto error; - } - else if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + package = _PyDict_GetItemId(globals, &PyId___package__); + if (package != NULL && package != Py_None) { + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; + } } - Py_INCREF(package); - - if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); - if (borrowed_dot == NULL) { + else { + if (PyErr_WarnEx(PyExc_ImportWarning, + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", 1) < 0) { goto error; } - partition = PyUnicode_RPartition(package, borrowed_dot); - Py_DECREF(package); - if (partition == NULL) { + + package = _PyDict_GetItemId(globals, &PyId___name__); + if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); goto error; } - package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); - Py_DECREF(partition); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + goto error; + } + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { + goto error; + } + partition = PyUnicode_RPartition(package, borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; + } + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); + } } } diff --git a/Python/importlib.h b/Python/importlib.h index a4daf62..709c5fd 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1057,7 +1057,7 @@ const unsigned char _Py_M__importlib[] = { 100,101,114,115,32,100,101,102,105,110,105,110,103,32,101,120, 101,99,95,109,111,100,117,108,101,40,41,32,109,117,115,116, 32,97,108,115,111,32,100,101,102,105,110,101,32,99,114,101, - 97,116,101,95,109,111,100,117,108,101,40,41,90,10,115,116, + 97,116,101,95,109,111,100,117,108,101,40,41,218,10,115,116, 97,99,107,108,101,118,101,108,233,2,0,0,0,41,9,114, 4,0,0,0,114,99,0,0,0,114,138,0,0,0,218,9, 95,119,97,114,110,105,110,103,115,218,4,119,97,114,110,218, @@ -1067,7 +1067,7 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,10,0,0,0,114,11,0,0,0,218,16,109, 111,100,117,108,101,95,102,114,111,109,95,115,112,101,99,58, 2,0,0,115,20,0,0,0,0,3,6,1,18,3,21,1, - 18,1,9,2,13,1,12,1,15,1,13,1,114,144,0,0, + 18,1,9,2,13,1,12,1,15,1,13,1,114,145,0,0, 0,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, 0,0,67,0,0,0,115,149,0,0,0,124,0,0,106,0, 0,100,1,0,107,8,0,114,21,0,100,2,0,110,6,0, @@ -1149,7 +1149,7 @@ const unsigned char _Py_M__importlib[] = { 0,1,1,1,89,110,1,0,88,124,1,0,83,41,7,78, 114,91,0,0,0,114,134,0,0,0,114,131,0,0,0,114, 121,0,0,0,114,33,0,0,0,114,95,0,0,0,41,13, - 114,99,0,0,0,114,146,0,0,0,114,15,0,0,0,114, + 114,99,0,0,0,114,147,0,0,0,114,15,0,0,0,114, 14,0,0,0,114,21,0,0,0,114,6,0,0,0,114,91, 0,0,0,114,96,0,0,0,114,1,0,0,0,114,134,0, 0,0,114,4,0,0,0,114,122,0,0,0,114,95,0,0, @@ -1159,7 +1159,7 @@ const unsigned char _Py_M__importlib[] = { 112,97,116,105,98,108,101,118,2,0,0,115,40,0,0,0, 0,4,19,2,16,1,24,1,3,1,16,1,13,1,5,1, 24,1,3,4,12,1,15,1,29,1,13,1,5,1,24,1, - 3,1,13,1,13,1,5,1,114,148,0,0,0,99,1,0, + 3,1,13,1,13,1,5,1,114,149,0,0,0,99,1,0, 0,0,0,0,0,0,2,0,0,0,11,0,0,0,67,0, 0,0,115,159,0,0,0,124,0,0,106,0,0,100,0,0, 107,9,0,114,43,0,116,1,0,124,0,0,106,0,0,100, @@ -1174,14 +1174,14 @@ const unsigned char _Py_M__importlib[] = { 0,106,7,0,25,83,41,4,78,114,139,0,0,0,122,14, 109,105,115,115,105,110,103,32,108,111,97,100,101,114,114,15, 0,0,0,41,11,114,99,0,0,0,114,4,0,0,0,114, - 148,0,0,0,114,144,0,0,0,114,102,0,0,0,114,110, + 149,0,0,0,114,145,0,0,0,114,102,0,0,0,114,110, 0,0,0,114,77,0,0,0,114,15,0,0,0,114,139,0, 0,0,114,14,0,0,0,114,21,0,0,0,41,2,114,88, 0,0,0,114,89,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,218,14,95,108,111,97,100,95,117, 110,108,111,99,107,101,100,147,2,0,0,115,20,0,0,0, 0,2,15,2,18,1,10,2,12,1,13,1,15,1,15,1, - 24,3,23,5,114,149,0,0,0,99,1,0,0,0,0,0, + 24,3,23,5,114,150,0,0,0,99,1,0,0,0,0,0, 0,0,1,0,0,0,9,0,0,0,67,0,0,0,115,47, 0,0,0,116,0,0,106,1,0,131,0,0,1,116,2,0, 124,0,0,106,3,0,131,1,0,143,15,0,1,116,4,0, @@ -1198,8 +1198,8 @@ const unsigned char _Py_M__importlib[] = { 100,117,108,101,115,44,32,116,104,97,116,32,101,120,105,115, 116,105,110,103,32,109,111,100,117,108,101,32,103,101,116,115, 10,32,32,32,32,99,108,111,98,98,101,114,101,100,46,10, - 10,32,32,32,32,78,41,5,114,57,0,0,0,114,145,0, - 0,0,114,54,0,0,0,114,15,0,0,0,114,149,0,0, + 10,32,32,32,32,78,41,5,114,57,0,0,0,114,146,0, + 0,0,114,54,0,0,0,114,15,0,0,0,114,150,0,0, 0,41,1,114,88,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,114,87,0,0,0,170,2,0,0, 115,6,0,0,0,0,9,10,1,16,1,114,87,0,0,0, @@ -1274,8 +1274,8 @@ const unsigned char _Py_M__importlib[] = { 112,114,101,99,97,116,101,100,46,32,32,85,115,101,32,102, 105,110,100,95,115,112,101,99,40,41,32,105,110,115,116,101, 97,100,46,10,10,32,32,32,32,32,32,32,32,78,41,2, - 114,154,0,0,0,114,99,0,0,0,41,4,114,151,0,0, - 0,114,78,0,0,0,114,152,0,0,0,114,88,0,0,0, + 114,155,0,0,0,114,99,0,0,0,41,4,114,152,0,0, + 0,114,78,0,0,0,114,153,0,0,0,114,88,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, 11,102,105,110,100,95,109,111,100,117,108,101,213,2,0,0, 115,4,0,0,0,0,9,18,1,122,27,66,117,105,108,116, @@ -1315,7 +1315,7 @@ const unsigned char _Py_M__importlib[] = { 32,97,115,32,98,117,105,108,116,45,105,110,32,109,111,100, 117,108,101,115,32,100,111,32,110,111,116,32,104,97,118,101, 32,99,111,100,101,32,111,98,106,101,99,116,115,46,78,114, - 10,0,0,0,41,2,114,151,0,0,0,114,78,0,0,0, + 10,0,0,0,41,2,114,152,0,0,0,114,78,0,0,0, 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, 8,103,101,116,95,99,111,100,101,238,2,0,0,115,2,0, 0,0,0,4,122,24,66,117,105,108,116,105,110,73,109,112, @@ -1326,7 +1326,7 @@ const unsigned char _Py_M__importlib[] = { 117,105,108,116,45,105,110,32,109,111,100,117,108,101,115,32, 100,111,32,110,111,116,32,104,97,118,101,32,115,111,117,114, 99,101,32,99,111,100,101,46,78,114,10,0,0,0,41,2, - 114,151,0,0,0,114,78,0,0,0,114,10,0,0,0,114, + 114,152,0,0,0,114,78,0,0,0,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,10,103,101,116,95,115, 111,117,114,99,101,244,2,0,0,115,2,0,0,0,0,4, 122,26,66,117,105,108,116,105,110,73,109,112,111,114,116,101, @@ -1336,7 +1336,7 @@ const unsigned char _Py_M__importlib[] = { 116,117,114,110,32,70,97,108,115,101,32,97,115,32,98,117, 105,108,116,45,105,110,32,109,111,100,117,108,101,115,32,97, 114,101,32,110,101,118,101,114,32,112,97,99,107,97,103,101, - 115,46,70,114,10,0,0,0,41,2,114,151,0,0,0,114, + 115,46,70,114,10,0,0,0,41,2,114,152,0,0,0,114, 78,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, 0,0,0,114,109,0,0,0,250,2,0,0,115,2,0,0, 0,0,4,122,26,66,117,105,108,116,105,110,73,109,112,111, @@ -1344,14 +1344,14 @@ const unsigned char _Py_M__importlib[] = { 17,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0, 114,3,0,0,0,218,12,115,116,97,116,105,99,109,101,116, 104,111,100,114,92,0,0,0,218,11,99,108,97,115,115,109, - 101,116,104,111,100,114,154,0,0,0,114,155,0,0,0,114, - 138,0,0,0,114,139,0,0,0,114,81,0,0,0,114,156, - 0,0,0,114,157,0,0,0,114,109,0,0,0,114,90,0, - 0,0,114,146,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,150,0,0,0, + 101,116,104,111,100,114,155,0,0,0,114,156,0,0,0,114, + 138,0,0,0,114,139,0,0,0,114,81,0,0,0,114,157, + 0,0,0,114,158,0,0,0,114,109,0,0,0,114,90,0, + 0,0,114,147,0,0,0,114,10,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,151,0,0,0, 186,2,0,0,115,30,0,0,0,12,7,6,2,18,9,3, 1,21,8,3,1,18,11,18,8,18,5,3,1,21,5,3, - 1,21,5,3,1,21,5,114,150,0,0,0,99,0,0,0, + 1,21,5,3,1,21,5,114,151,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,5,0,0,0,64,0,0, 0,115,211,0,0,0,101,0,0,90,1,0,100,0,0,90, 2,0,100,1,0,90,3,0,101,4,0,100,2,0,100,3, @@ -1399,9 +1399,9 @@ const unsigned char _Py_M__importlib[] = { 124,0,0,100,1,0,100,2,0,131,2,1,83,100,0,0, 83,100,0,0,83,41,3,78,114,107,0,0,0,90,6,102, 114,111,122,101,110,41,3,114,57,0,0,0,114,82,0,0, - 0,114,85,0,0,0,41,4,114,151,0,0,0,114,78,0, - 0,0,114,152,0,0,0,114,153,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,154,0,0,0, + 0,114,85,0,0,0,41,4,114,152,0,0,0,114,78,0, + 0,0,114,153,0,0,0,114,154,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,155,0,0,0, 21,3,0,0,115,6,0,0,0,0,2,15,1,19,2,122, 24,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, 102,105,110,100,95,115,112,101,99,99,3,0,0,0,0,0, @@ -1414,9 +1414,9 @@ const unsigned char _Py_M__importlib[] = { 101,99,97,116,101,100,46,32,32,85,115,101,32,102,105,110, 100,95,115,112,101,99,40,41,32,105,110,115,116,101,97,100, 46,10,10,32,32,32,32,32,32,32,32,78,41,2,114,57, - 0,0,0,114,82,0,0,0,41,3,114,151,0,0,0,114, - 78,0,0,0,114,152,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,155,0,0,0,28,3,0, + 0,0,0,114,82,0,0,0,41,3,114,152,0,0,0,114, + 78,0,0,0,114,153,0,0,0,114,10,0,0,0,114,10, + 0,0,0,114,11,0,0,0,114,156,0,0,0,28,3,0, 0,115,2,0,0,0,0,7,122,26,70,114,111,122,101,110, 73,109,112,111,114,116,101,114,46,102,105,110,100,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, @@ -1424,7 +1424,7 @@ const unsigned char _Py_M__importlib[] = { 0,83,41,2,122,42,85,115,101,32,100,101,102,97,117,108, 116,32,115,101,109,97,110,116,105,99,115,32,102,111,114,32, 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, - 78,114,10,0,0,0,41,2,114,151,0,0,0,114,88,0, + 78,114,10,0,0,0,41,2,114,152,0,0,0,114,88,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, 0,114,138,0,0,0,37,3,0,0,115,0,0,0,0,122, 28,70,114,111,122,101,110,73,109,112,111,114,116,101,114,46, @@ -1457,8 +1457,8 @@ const unsigned char _Py_M__importlib[] = { 99,97,116,101,100,46,32,32,85,115,101,32,101,120,101,99, 95,109,111,100,117,108,101,40,41,32,105,110,115,116,101,97, 100,46,10,10,32,32,32,32,32,32,32,32,41,1,114,90, - 0,0,0,41,2,114,151,0,0,0,114,78,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,146, + 0,0,0,41,2,114,152,0,0,0,114,78,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,147, 0,0,0,50,3,0,0,115,2,0,0,0,0,7,122,26, 70,114,111,122,101,110,73,109,112,111,114,116,101,114,46,108, 111,97,100,95,109,111,100,117,108,101,99,2,0,0,0,0, @@ -1467,9 +1467,9 @@ const unsigned char _Py_M__importlib[] = { 83,41,1,122,45,82,101,116,117,114,110,32,116,104,101,32, 99,111,100,101,32,111,98,106,101,99,116,32,102,111,114,32, 116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108, - 101,46,41,2,114,57,0,0,0,114,162,0,0,0,41,2, - 114,151,0,0,0,114,78,0,0,0,114,10,0,0,0,114, - 10,0,0,0,114,11,0,0,0,114,156,0,0,0,59,3, + 101,46,41,2,114,57,0,0,0,114,163,0,0,0,41,2, + 114,152,0,0,0,114,78,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,114,157,0,0,0,59,3, 0,0,115,2,0,0,0,0,4,122,23,70,114,111,122,101, 110,73,109,112,111,114,116,101,114,46,103,101,116,95,99,111, 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1478,8 +1478,8 @@ const unsigned char _Py_M__importlib[] = { 97,115,32,102,114,111,122,101,110,32,109,111,100,117,108,101, 115,32,100,111,32,110,111,116,32,104,97,118,101,32,115,111, 117,114,99,101,32,99,111,100,101,46,78,114,10,0,0,0, - 41,2,114,151,0,0,0,114,78,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,114,157,0,0,0, + 41,2,114,152,0,0,0,114,78,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,11,0,0,0,114,158,0,0,0, 65,3,0,0,115,2,0,0,0,0,4,122,25,70,114,111, 122,101,110,73,109,112,111,114,116,101,114,46,103,101,116,95, 115,111,117,114,99,101,99,2,0,0,0,0,0,0,0,2, @@ -1489,21 +1489,21 @@ const unsigned char _Py_M__importlib[] = { 116,104,101,32,102,114,111,122,101,110,32,109,111,100,117,108, 101,32,105,115,32,97,32,112,97,99,107,97,103,101,46,41, 2,114,57,0,0,0,90,17,105,115,95,102,114,111,122,101, - 110,95,112,97,99,107,97,103,101,41,2,114,151,0,0,0, + 110,95,112,97,99,107,97,103,101,41,2,114,152,0,0,0, 114,78,0,0,0,114,10,0,0,0,114,10,0,0,0,114, 11,0,0,0,114,109,0,0,0,71,3,0,0,115,2,0, 0,0,0,4,122,25,70,114,111,122,101,110,73,109,112,111, 114,116,101,114,46,105,115,95,112,97,99,107,97,103,101,41, 16,114,1,0,0,0,114,0,0,0,0,114,2,0,0,0, - 114,3,0,0,0,114,158,0,0,0,114,92,0,0,0,114, - 159,0,0,0,114,154,0,0,0,114,155,0,0,0,114,138, - 0,0,0,114,139,0,0,0,114,146,0,0,0,114,84,0, - 0,0,114,156,0,0,0,114,157,0,0,0,114,109,0,0, + 114,3,0,0,0,114,159,0,0,0,114,92,0,0,0,114, + 160,0,0,0,114,155,0,0,0,114,156,0,0,0,114,138, + 0,0,0,114,139,0,0,0,114,147,0,0,0,114,84,0, + 0,0,114,157,0,0,0,114,158,0,0,0,114,109,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,114,160,0,0,0,3,3,0,0,115,30, + 114,11,0,0,0,114,161,0,0,0,3,3,0,0,115,30, 0,0,0,12,7,6,2,18,9,3,1,21,6,3,1,18, 8,18,4,18,9,18,9,3,1,21,5,3,1,21,5,3, - 1,114,160,0,0,0,99,0,0,0,0,0,0,0,0,0, + 1,114,161,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,2,0,0,0,64,0,0,0,115,46,0,0,0, 101,0,0,90,1,0,100,0,0,90,2,0,100,1,0,90, 3,0,100,2,0,100,3,0,132,0,0,90,4,0,100,4, @@ -1516,7 +1516,7 @@ const unsigned char _Py_M__importlib[] = { 14,0,0,0,116,0,0,106,1,0,131,0,0,1,100,1, 0,83,41,2,122,24,65,99,113,117,105,114,101,32,116,104, 101,32,105,109,112,111,114,116,32,108,111,99,107,46,78,41, - 2,114,57,0,0,0,114,145,0,0,0,41,1,114,19,0, + 2,114,57,0,0,0,114,146,0,0,0,41,1,114,19,0, 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, 0,114,23,0,0,0,84,3,0,0,115,2,0,0,0,0, 2,122,28,95,73,109,112,111,114,116,76,111,99,107,67,111, @@ -1538,8 +1538,8 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,0,0,0,0,114,2,0,0,0,114,3,0, 0,0,114,23,0,0,0,114,30,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,165,0,0,0,80,3,0,0,115,6,0,0,0,12,2, - 6,2,12,4,114,165,0,0,0,99,3,0,0,0,0,0, + 114,166,0,0,0,80,3,0,0,115,6,0,0,0,12,2, + 6,2,12,4,114,166,0,0,0,99,3,0,0,0,0,0, 0,0,5,0,0,0,4,0,0,0,67,0,0,0,115,88, 0,0,0,124,1,0,106,0,0,100,1,0,124,2,0,100, 2,0,24,131,2,0,125,3,0,116,1,0,124,3,0,131, @@ -1562,17 +1562,17 @@ const unsigned char _Py_M__importlib[] = { 0,0,0,114,10,0,0,0,114,11,0,0,0,218,13,95, 114,101,115,111,108,118,101,95,110,97,109,101,93,3,0,0, 115,10,0,0,0,0,2,22,1,18,1,12,1,10,1,114, - 171,0,0,0,99,3,0,0,0,0,0,0,0,4,0,0, + 172,0,0,0,99,3,0,0,0,0,0,0,0,4,0,0, 0,3,0,0,0,67,0,0,0,115,47,0,0,0,124,0, 0,106,0,0,124,1,0,124,2,0,131,2,0,125,3,0, 124,3,0,100,0,0,107,8,0,114,34,0,100,0,0,83, 116,1,0,124,1,0,124,3,0,131,2,0,83,41,1,78, - 41,2,114,155,0,0,0,114,85,0,0,0,41,4,218,6, - 102,105,110,100,101,114,114,15,0,0,0,114,152,0,0,0, + 41,2,114,156,0,0,0,114,85,0,0,0,41,4,218,6, + 102,105,110,100,101,114,114,15,0,0,0,114,153,0,0,0, 114,99,0,0,0,114,10,0,0,0,114,10,0,0,0,114, 11,0,0,0,218,17,95,102,105,110,100,95,115,112,101,99, 95,108,101,103,97,99,121,102,3,0,0,115,8,0,0,0, - 0,3,18,1,12,1,4,1,114,173,0,0,0,99,3,0, + 0,3,18,1,12,1,4,1,114,174,0,0,0,99,3,0, 0,0,0,0,0,0,9,0,0,0,27,0,0,0,67,0, 0,0,115,42,1,0,0,116,0,0,106,1,0,100,1,0, 107,9,0,114,41,0,116,0,0,106,1,0,12,114,41,0, @@ -1597,19 +1597,19 @@ const unsigned char _Py_M__importlib[] = { 108,101,39,115,32,108,111,97,100,101,114,46,78,122,22,115, 121,115,46,109,101,116,97,95,112,97,116,104,32,105,115,32, 101,109,112,116,121,41,11,114,14,0,0,0,218,9,109,101, - 116,97,95,112,97,116,104,114,141,0,0,0,114,142,0,0, + 116,97,95,112,97,116,104,114,142,0,0,0,114,143,0,0, 0,218,13,73,109,112,111,114,116,87,97,114,110,105,110,103, - 114,21,0,0,0,114,165,0,0,0,114,154,0,0,0,114, - 96,0,0,0,114,173,0,0,0,114,95,0,0,0,41,9, - 114,15,0,0,0,114,152,0,0,0,114,153,0,0,0,90, - 9,105,115,95,114,101,108,111,97,100,114,172,0,0,0,114, - 154,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, + 114,21,0,0,0,114,166,0,0,0,114,155,0,0,0,114, + 96,0,0,0,114,174,0,0,0,114,95,0,0,0,41,9, + 114,15,0,0,0,114,153,0,0,0,114,154,0,0,0,90, + 9,105,115,95,114,101,108,111,97,100,114,173,0,0,0,114, + 155,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, 0,0,218,10,95,102,105,110,100,95,115,112,101,99,111,3, 0,0,115,48,0,0,0,0,2,25,1,16,4,15,1,16, 1,10,1,3,1,13,1,13,1,18,1,12,1,8,2,25, 1,12,2,22,1,13,1,3,1,13,1,13,4,9,2,12, - 1,4,2,7,2,8,2,114,176,0,0,0,99,3,0,0, + 1,4,2,7,2,8,2,114,177,0,0,0,99,3,0,0, 0,0,0,0,0,4,0,0,0,4,0,0,0,67,0,0, 0,115,179,0,0,0,116,0,0,124,0,0,116,1,0,131, 2,0,115,42,0,116,2,0,100,1,0,106,3,0,116,4, @@ -1638,14 +1638,14 @@ const unsigned char _Py_M__importlib[] = { 101,32,110,97,109,101,78,41,9,218,10,105,115,105,110,115, 116,97,110,99,101,218,3,115,116,114,218,9,84,121,112,101, 69,114,114,111,114,114,50,0,0,0,114,13,0,0,0,114, - 168,0,0,0,114,14,0,0,0,114,21,0,0,0,218,11, + 169,0,0,0,114,14,0,0,0,114,21,0,0,0,218,11, 83,121,115,116,101,109,69,114,114,111,114,41,4,114,15,0, - 0,0,114,169,0,0,0,114,170,0,0,0,114,147,0,0, + 0,0,114,170,0,0,0,114,171,0,0,0,114,148,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,13,95,115,97,110,105,116,121,95,99,104,101,99,107,151, 3,0,0,115,24,0,0,0,0,2,15,1,27,1,12,1, 12,1,6,1,15,1,15,1,15,1,6,2,21,1,19,1, - 114,181,0,0,0,122,16,78,111,32,109,111,100,117,108,101, + 114,182,0,0,0,122,16,78,111,32,109,111,100,117,108,101, 32,110,97,109,101,100,32,122,4,123,33,114,125,99,2,0, 0,0,0,0,0,0,8,0,0,0,12,0,0,0,67,0, 0,0,115,40,1,0,0,100,0,0,125,2,0,124,0,0, @@ -1669,21 +1669,21 @@ const unsigned char _Py_M__importlib[] = { 100,5,0,25,124,7,0,131,3,0,1,124,7,0,83,41, 6,78,114,121,0,0,0,114,33,0,0,0,122,23,59,32, 123,33,114,125,32,105,115,32,110,111,116,32,97,32,112,97, - 99,107,97,103,101,114,15,0,0,0,114,140,0,0,0,41, + 99,107,97,103,101,114,15,0,0,0,114,141,0,0,0,41, 12,114,122,0,0,0,114,14,0,0,0,114,21,0,0,0, 114,65,0,0,0,114,131,0,0,0,114,96,0,0,0,218, 8,95,69,82,82,95,77,83,71,114,50,0,0,0,114,77, - 0,0,0,114,176,0,0,0,114,149,0,0,0,114,5,0, + 0,0,0,114,177,0,0,0,114,150,0,0,0,114,5,0, 0,0,41,8,114,15,0,0,0,218,7,105,109,112,111,114, - 116,95,114,152,0,0,0,114,123,0,0,0,90,13,112,97, - 114,101,110,116,95,109,111,100,117,108,101,114,147,0,0,0, + 116,95,114,153,0,0,0,114,123,0,0,0,90,13,112,97, + 114,101,110,116,95,109,111,100,117,108,101,114,148,0,0,0, 114,88,0,0,0,114,89,0,0,0,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,23,95,102,105,110,100, 95,97,110,100,95,108,111,97,100,95,117,110,108,111,99,107, 101,100,171,3,0,0,115,42,0,0,0,0,1,6,1,19, 1,6,1,15,1,13,2,15,1,11,1,13,1,3,1,13, 1,13,1,22,1,26,1,15,1,12,1,30,2,12,1,6, - 2,13,1,29,1,114,184,0,0,0,99,2,0,0,0,0, + 2,13,1,29,1,114,185,0,0,0,99,2,0,0,0,0, 0,0,0,2,0,0,0,10,0,0,0,67,0,0,0,115, 37,0,0,0,116,0,0,124,0,0,131,1,0,143,18,0, 1,116,1,0,124,0,0,124,1,0,131,2,0,83,87,100, @@ -1691,11 +1691,11 @@ const unsigned char _Py_M__importlib[] = { 100,32,97,110,100,32,108,111,97,100,32,116,104,101,32,109, 111,100,117,108,101,44,32,97,110,100,32,114,101,108,101,97, 115,101,32,116,104,101,32,105,109,112,111,114,116,32,108,111, - 99,107,46,78,41,2,114,54,0,0,0,114,184,0,0,0, - 41,2,114,15,0,0,0,114,183,0,0,0,114,10,0,0, + 99,107,46,78,41,2,114,54,0,0,0,114,185,0,0,0, + 41,2,114,15,0,0,0,114,184,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,11,0,0,0,218,14,95,102,105, 110,100,95,97,110,100,95,108,111,97,100,198,3,0,0,115, - 4,0,0,0,0,2,13,1,114,185,0,0,0,114,33,0, + 4,0,0,0,0,2,13,1,114,186,0,0,0,114,33,0, 0,0,99,3,0,0,0,0,0,0,0,5,0,0,0,4, 0,0,0,67,0,0,0,115,166,0,0,0,116,0,0,124, 0,0,124,1,0,124,2,0,131,3,0,1,124,2,0,100, @@ -1731,16 +1731,16 @@ const unsigned char _Py_M__importlib[] = { 78,122,40,105,109,112,111,114,116,32,111,102,32,123,125,32, 104,97,108,116,101,100,59,32,78,111,110,101,32,105,110,32, 115,121,115,46,109,111,100,117,108,101,115,114,15,0,0,0, - 41,12,114,181,0,0,0,114,171,0,0,0,114,57,0,0, - 0,114,145,0,0,0,114,14,0,0,0,114,21,0,0,0, - 114,185,0,0,0,218,11,95,103,99,100,95,105,109,112,111, + 41,12,114,182,0,0,0,114,172,0,0,0,114,57,0,0, + 0,114,146,0,0,0,114,14,0,0,0,114,21,0,0,0, + 114,186,0,0,0,218,11,95,103,99,100,95,105,109,112,111, 114,116,114,58,0,0,0,114,50,0,0,0,114,77,0,0, - 0,114,63,0,0,0,41,5,114,15,0,0,0,114,169,0, - 0,0,114,170,0,0,0,114,89,0,0,0,114,74,0,0, + 0,114,63,0,0,0,41,5,114,15,0,0,0,114,170,0, + 0,0,114,171,0,0,0,114,89,0,0,0,114,74,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 114,186,0,0,0,204,3,0,0,115,28,0,0,0,0,9, + 114,187,0,0,0,204,3,0,0,115,28,0,0,0,0,9, 16,1,12,1,18,1,10,1,15,1,13,1,13,1,12,1, - 10,1,6,1,9,1,18,1,10,1,114,186,0,0,0,99, + 10,1,6,1,9,1,18,1,10,1,114,187,0,0,0,99, 3,0,0,0,0,0,0,0,6,0,0,0,17,0,0,0, 67,0,0,0,115,239,0,0,0,116,0,0,124,0,0,100, 1,0,131,2,0,114,235,0,100,2,0,124,1,0,107,6, @@ -1776,213 +1776,226 @@ const unsigned char _Py_M__importlib[] = { 1,42,218,7,95,95,97,108,108,95,95,122,5,123,125,46, 123,125,78,41,13,114,4,0,0,0,114,130,0,0,0,218, 6,114,101,109,111,118,101,218,6,101,120,116,101,110,100,114, - 188,0,0,0,114,50,0,0,0,114,1,0,0,0,114,65, - 0,0,0,114,77,0,0,0,114,178,0,0,0,114,71,0, + 189,0,0,0,114,50,0,0,0,114,1,0,0,0,114,65, + 0,0,0,114,77,0,0,0,114,179,0,0,0,114,71,0, 0,0,218,15,95,69,82,82,95,77,83,71,95,80,82,69, 70,73,88,114,15,0,0,0,41,6,114,89,0,0,0,218, - 8,102,114,111,109,108,105,115,116,114,183,0,0,0,218,1, + 8,102,114,111,109,108,105,115,116,114,184,0,0,0,218,1, 120,90,9,102,114,111,109,95,110,97,109,101,90,3,101,120, 99,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,16,95,104,97,110,100,108,101,95,102,114,111,109,108,105, 115,116,228,3,0,0,115,34,0,0,0,0,10,15,1,12, 1,12,1,13,1,15,1,16,1,13,1,15,1,21,1,3, - 1,17,1,18,4,21,1,15,1,3,1,26,1,114,194,0, - 0,0,99,1,0,0,0,0,0,0,0,2,0,0,0,2, - 0,0,0,67,0,0,0,115,72,0,0,0,124,0,0,106, + 1,17,1,18,4,21,1,15,1,3,1,26,1,114,195,0, + 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,5, + 0,0,0,67,0,0,0,115,128,0,0,0,124,0,0,106, 0,0,100,1,0,131,1,0,125,1,0,124,1,0,100,2, - 0,107,8,0,114,68,0,124,0,0,100,3,0,25,125,1, - 0,100,4,0,124,0,0,107,7,0,114,68,0,124,1,0, - 106,1,0,100,5,0,131,1,0,100,6,0,25,125,1,0, - 124,1,0,83,41,7,122,167,67,97,108,99,117,108,97,116, - 101,32,119,104,97,116,32,95,95,112,97,99,107,97,103,101, - 95,95,32,115,104,111,117,108,100,32,98,101,46,10,10,32, - 32,32,32,95,95,112,97,99,107,97,103,101,95,95,32,105, - 115,32,110,111,116,32,103,117,97,114,97,110,116,101,101,100, - 32,116,111,32,98,101,32,100,101,102,105,110,101,100,32,111, - 114,32,99,111,117,108,100,32,98,101,32,115,101,116,32,116, - 111,32,78,111,110,101,10,32,32,32,32,116,111,32,114,101, - 112,114,101,115,101,110,116,32,116,104,97,116,32,105,116,115, - 32,112,114,111,112,101,114,32,118,97,108,117,101,32,105,115, - 32,117,110,107,110,111,119,110,46,10,10,32,32,32,32,114, - 134,0,0,0,78,114,1,0,0,0,114,131,0,0,0,114, - 121,0,0,0,114,33,0,0,0,41,2,114,42,0,0,0, - 114,122,0,0,0,41,2,218,7,103,108,111,98,97,108,115, - 114,169,0,0,0,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,17,95,99,97,108,99,95,95,95,112,97, - 99,107,97,103,101,95,95,4,4,0,0,115,12,0,0,0, - 0,7,15,1,12,1,10,1,12,1,19,1,114,196,0,0, - 0,99,5,0,0,0,0,0,0,0,9,0,0,0,5,0, - 0,0,67,0,0,0,115,227,0,0,0,124,4,0,100,1, - 0,107,2,0,114,27,0,116,0,0,124,0,0,131,1,0, - 125,5,0,110,54,0,124,1,0,100,2,0,107,9,0,114, - 45,0,124,1,0,110,3,0,105,0,0,125,6,0,116,1, - 0,124,6,0,131,1,0,125,7,0,116,0,0,124,0,0, - 124,7,0,124,4,0,131,3,0,125,5,0,124,3,0,115, - 207,0,124,4,0,100,1,0,107,2,0,114,122,0,116,0, - 0,124,0,0,106,2,0,100,3,0,131,1,0,100,1,0, - 25,131,1,0,83,124,0,0,115,132,0,124,5,0,83,116, - 3,0,124,0,0,131,1,0,116,3,0,124,0,0,106,2, - 0,100,3,0,131,1,0,100,1,0,25,131,1,0,24,125, - 8,0,116,4,0,106,5,0,124,5,0,106,6,0,100,2, - 0,116,3,0,124,5,0,106,6,0,131,1,0,124,8,0, - 24,133,2,0,25,25,83,110,16,0,116,7,0,124,5,0, - 124,3,0,116,0,0,131,3,0,83,100,2,0,83,41,4, - 97,214,1,0,0,73,109,112,111,114,116,32,97,32,109,111, - 100,117,108,101,46,10,10,32,32,32,32,84,104,101,32,39, - 103,108,111,98,97,108,115,39,32,97,114,103,117,109,101,110, - 116,32,105,115,32,117,115,101,100,32,116,111,32,105,110,102, - 101,114,32,119,104,101,114,101,32,116,104,101,32,105,109,112, - 111,114,116,32,105,115,32,111,99,99,117,114,105,110,103,32, - 102,114,111,109,10,32,32,32,32,116,111,32,104,97,110,100, - 108,101,32,114,101,108,97,116,105,118,101,32,105,109,112,111, - 114,116,115,46,32,84,104,101,32,39,108,111,99,97,108,115, - 39,32,97,114,103,117,109,101,110,116,32,105,115,32,105,103, - 110,111,114,101,100,46,32,84,104,101,10,32,32,32,32,39, - 102,114,111,109,108,105,115,116,39,32,97,114,103,117,109,101, - 110,116,32,115,112,101,99,105,102,105,101,115,32,119,104,97, - 116,32,115,104,111,117,108,100,32,101,120,105,115,116,32,97, - 115,32,97,116,116,114,105,98,117,116,101,115,32,111,110,32, - 116,104,101,32,109,111,100,117,108,101,10,32,32,32,32,98, - 101,105,110,103,32,105,109,112,111,114,116,101,100,32,40,101, - 46,103,46,32,96,96,102,114,111,109,32,109,111,100,117,108, - 101,32,105,109,112,111,114,116,32,60,102,114,111,109,108,105, - 115,116,62,96,96,41,46,32,32,84,104,101,32,39,108,101, - 118,101,108,39,10,32,32,32,32,97,114,103,117,109,101,110, - 116,32,114,101,112,114,101,115,101,110,116,115,32,116,104,101, - 32,112,97,99,107,97,103,101,32,108,111,99,97,116,105,111, - 110,32,116,111,32,105,109,112,111,114,116,32,102,114,111,109, - 32,105,110,32,97,32,114,101,108,97,116,105,118,101,10,32, - 32,32,32,105,109,112,111,114,116,32,40,101,46,103,46,32, - 96,96,102,114,111,109,32,46,46,112,107,103,32,105,109,112, - 111,114,116,32,109,111,100,96,96,32,119,111,117,108,100,32, - 104,97,118,101,32,97,32,39,108,101,118,101,108,39,32,111, - 102,32,50,41,46,10,10,32,32,32,32,114,33,0,0,0, - 78,114,121,0,0,0,41,8,114,186,0,0,0,114,196,0, - 0,0,218,9,112,97,114,116,105,116,105,111,110,114,167,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,1,0,0, - 0,114,194,0,0,0,41,9,114,15,0,0,0,114,195,0, - 0,0,218,6,108,111,99,97,108,115,114,192,0,0,0,114, - 170,0,0,0,114,89,0,0,0,90,8,103,108,111,98,97, - 108,115,95,114,169,0,0,0,90,7,99,117,116,95,111,102, - 102,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, - 218,10,95,95,105,109,112,111,114,116,95,95,19,4,0,0, - 115,26,0,0,0,0,11,12,1,15,2,24,1,12,1,18, - 1,6,3,12,1,23,1,6,1,4,4,35,3,40,2,114, - 199,0,0,0,99,1,0,0,0,0,0,0,0,2,0,0, - 0,3,0,0,0,67,0,0,0,115,53,0,0,0,116,0, - 0,106,1,0,124,0,0,131,1,0,125,1,0,124,1,0, - 100,0,0,107,8,0,114,43,0,116,2,0,100,1,0,124, - 0,0,23,131,1,0,130,1,0,116,3,0,124,1,0,131, - 1,0,83,41,2,78,122,25,110,111,32,98,117,105,108,116, - 45,105,110,32,109,111,100,117,108,101,32,110,97,109,101,100, - 32,41,4,114,150,0,0,0,114,154,0,0,0,114,77,0, - 0,0,114,149,0,0,0,41,2,114,15,0,0,0,114,88, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,18,95,98,117,105,108,116,105,110,95,102,114,111, - 109,95,110,97,109,101,54,4,0,0,115,8,0,0,0,0, - 1,15,1,12,1,16,1,114,200,0,0,0,99,2,0,0, - 0,0,0,0,0,12,0,0,0,12,0,0,0,67,0,0, - 0,115,74,1,0,0,124,1,0,97,0,0,124,0,0,97, - 1,0,116,2,0,116,1,0,131,1,0,125,2,0,120,123, - 0,116,1,0,106,3,0,106,4,0,131,0,0,68,93,106, - 0,92,2,0,125,3,0,125,4,0,116,5,0,124,4,0, - 124,2,0,131,2,0,114,40,0,124,3,0,116,1,0,106, - 6,0,107,6,0,114,91,0,116,7,0,125,5,0,110,27, - 0,116,0,0,106,8,0,124,3,0,131,1,0,114,40,0, - 116,9,0,125,5,0,110,3,0,113,40,0,116,10,0,124, - 4,0,124,5,0,131,2,0,125,6,0,116,11,0,124,6, - 0,124,4,0,131,2,0,1,113,40,0,87,116,1,0,106, - 3,0,116,12,0,25,125,7,0,120,73,0,100,5,0,68, - 93,65,0,125,8,0,124,8,0,116,1,0,106,3,0,107, - 7,0,114,206,0,116,13,0,124,8,0,131,1,0,125,9, - 0,110,13,0,116,1,0,106,3,0,124,8,0,25,125,9, - 0,116,14,0,124,7,0,124,8,0,124,9,0,131,3,0, - 1,113,170,0,87,121,16,0,116,13,0,100,2,0,131,1, - 0,125,10,0,87,110,24,0,4,116,15,0,107,10,0,114, - 25,1,1,1,1,100,3,0,125,10,0,89,110,1,0,88, - 116,14,0,124,7,0,100,2,0,124,10,0,131,3,0,1, - 116,13,0,100,4,0,131,1,0,125,11,0,116,14,0,124, - 7,0,100,4,0,124,11,0,131,3,0,1,100,3,0,83, - 41,6,122,250,83,101,116,117,112,32,105,109,112,111,114,116, - 108,105,98,32,98,121,32,105,109,112,111,114,116,105,110,103, - 32,110,101,101,100,101,100,32,98,117,105,108,116,45,105,110, - 32,109,111,100,117,108,101,115,32,97,110,100,32,105,110,106, - 101,99,116,105,110,103,32,116,104,101,109,10,32,32,32,32, - 105,110,116,111,32,116,104,101,32,103,108,111,98,97,108,32, - 110,97,109,101,115,112,97,99,101,46,10,10,32,32,32,32, - 65,115,32,115,121,115,32,105,115,32,110,101,101,100,101,100, - 32,102,111,114,32,115,121,115,46,109,111,100,117,108,101,115, - 32,97,99,99,101,115,115,32,97,110,100,32,95,105,109,112, - 32,105,115,32,110,101,101,100,101,100,32,116,111,32,108,111, - 97,100,32,98,117,105,108,116,45,105,110,10,32,32,32,32, - 109,111,100,117,108,101,115,44,32,116,104,111,115,101,32,116, - 119,111,32,109,111,100,117,108,101,115,32,109,117,115,116,32, - 98,101,32,101,120,112,108,105,99,105,116,108,121,32,112,97, - 115,115,101,100,32,105,110,46,10,10,32,32,32,32,114,141, - 0,0,0,114,34,0,0,0,78,114,62,0,0,0,41,1, - 122,9,95,119,97,114,110,105,110,103,115,41,16,114,57,0, - 0,0,114,14,0,0,0,114,13,0,0,0,114,21,0,0, - 0,218,5,105,116,101,109,115,114,177,0,0,0,114,76,0, - 0,0,114,150,0,0,0,114,82,0,0,0,114,160,0,0, - 0,114,132,0,0,0,114,137,0,0,0,114,1,0,0,0, - 114,200,0,0,0,114,5,0,0,0,114,77,0,0,0,41, - 12,218,10,115,121,115,95,109,111,100,117,108,101,218,11,95, - 105,109,112,95,109,111,100,117,108,101,90,11,109,111,100,117, - 108,101,95,116,121,112,101,114,15,0,0,0,114,89,0,0, - 0,114,99,0,0,0,114,88,0,0,0,90,11,115,101,108, - 102,95,109,111,100,117,108,101,90,12,98,117,105,108,116,105, - 110,95,110,97,109,101,90,14,98,117,105,108,116,105,110,95, - 109,111,100,117,108,101,90,13,116,104,114,101,97,100,95,109, - 111,100,117,108,101,90,14,119,101,97,107,114,101,102,95,109, - 111,100,117,108,101,114,10,0,0,0,114,10,0,0,0,114, - 11,0,0,0,218,6,95,115,101,116,117,112,61,4,0,0, - 115,50,0,0,0,0,9,6,1,6,3,12,1,28,1,15, - 1,15,1,9,1,15,1,9,2,3,1,15,1,17,3,13, - 1,13,1,15,1,15,2,13,1,20,3,3,1,16,1,13, - 2,11,1,16,3,12,1,114,204,0,0,0,99,2,0,0, - 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, - 0,115,87,0,0,0,116,0,0,124,0,0,124,1,0,131, - 2,0,1,116,1,0,106,2,0,106,3,0,116,4,0,131, - 1,0,1,116,1,0,106,2,0,106,3,0,116,5,0,131, - 1,0,1,100,1,0,100,2,0,108,6,0,125,2,0,124, - 2,0,97,7,0,124,2,0,106,8,0,116,1,0,106,9, - 0,116,10,0,25,131,1,0,1,100,2,0,83,41,3,122, - 50,73,110,115,116,97,108,108,32,105,109,112,111,114,116,108, - 105,98,32,97,115,32,116,104,101,32,105,109,112,108,101,109, - 101,110,116,97,116,105,111,110,32,111,102,32,105,109,112,111, - 114,116,46,114,33,0,0,0,78,41,11,114,204,0,0,0, - 114,14,0,0,0,114,174,0,0,0,114,113,0,0,0,114, - 150,0,0,0,114,160,0,0,0,218,26,95,102,114,111,122, - 101,110,95,105,109,112,111,114,116,108,105,98,95,101,120,116, - 101,114,110,97,108,114,119,0,0,0,218,8,95,105,110,115, - 116,97,108,108,114,21,0,0,0,114,1,0,0,0,41,3, - 114,202,0,0,0,114,203,0,0,0,114,205,0,0,0,114, - 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,206, - 0,0,0,108,4,0,0,115,12,0,0,0,0,2,13,2, - 16,1,16,3,12,1,6,1,114,206,0,0,0,41,51,114, - 3,0,0,0,114,119,0,0,0,114,12,0,0,0,114,16, - 0,0,0,114,17,0,0,0,114,59,0,0,0,114,41,0, - 0,0,114,48,0,0,0,114,31,0,0,0,114,32,0,0, - 0,114,53,0,0,0,114,54,0,0,0,114,56,0,0,0, - 114,63,0,0,0,114,65,0,0,0,114,75,0,0,0,114, - 81,0,0,0,114,84,0,0,0,114,90,0,0,0,114,101, - 0,0,0,114,102,0,0,0,114,106,0,0,0,114,85,0, - 0,0,218,6,111,98,106,101,99,116,90,9,95,80,79,80, - 85,76,65,84,69,114,132,0,0,0,114,137,0,0,0,114, - 144,0,0,0,114,97,0,0,0,114,86,0,0,0,114,148, - 0,0,0,114,149,0,0,0,114,87,0,0,0,114,150,0, - 0,0,114,160,0,0,0,114,165,0,0,0,114,171,0,0, - 0,114,173,0,0,0,114,176,0,0,0,114,181,0,0,0, - 114,191,0,0,0,114,182,0,0,0,114,184,0,0,0,114, - 185,0,0,0,114,186,0,0,0,114,194,0,0,0,114,196, - 0,0,0,114,199,0,0,0,114,200,0,0,0,114,204,0, - 0,0,114,206,0,0,0,114,10,0,0,0,114,10,0,0, - 0,114,10,0,0,0,114,11,0,0,0,218,8,60,109,111, - 100,117,108,101,62,8,0,0,0,115,96,0,0,0,6,17, - 6,2,12,8,12,4,19,20,6,2,6,3,22,4,19,68, - 19,21,19,19,12,19,12,19,12,11,18,8,12,11,12,12, - 12,16,12,36,19,27,19,101,24,26,9,3,18,45,18,60, - 12,18,12,17,12,25,12,29,12,23,12,16,19,73,19,77, - 19,13,12,9,12,9,15,40,12,17,6,1,10,2,12,27, - 12,6,18,24,12,32,12,15,24,35,12,7,12,47, + 0,107,9,0,114,34,0,124,1,0,106,1,0,83,124,0, + 0,106,0,0,100,3,0,131,1,0,125,2,0,124,2,0, + 100,2,0,107,8,0,114,124,0,116,2,0,106,3,0,100, + 4,0,116,4,0,100,5,0,100,6,0,131,2,1,1,124, + 0,0,100,7,0,25,125,2,0,100,8,0,124,0,0,107, + 7,0,114,124,0,124,2,0,106,5,0,100,9,0,131,1, + 0,100,10,0,25,125,2,0,124,2,0,83,41,11,122,167, + 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, + 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, + 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, + 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, + 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, + 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, + 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, + 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, + 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, + 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, + 46,10,10,32,32,32,32,114,95,0,0,0,78,114,134,0, + 0,0,122,89,99,97,110,39,116,32,114,101,115,111,108,118, + 101,32,112,97,99,107,97,103,101,32,102,114,111,109,32,95, + 95,115,112,101,99,95,95,32,111,114,32,95,95,112,97,99, + 107,97,103,101,95,95,44,32,102,97,108,108,105,110,103,32, + 98,97,99,107,32,111,110,32,95,95,110,97,109,101,95,95, + 32,97,110,100,32,95,95,112,97,116,104,95,95,114,140,0, + 0,0,233,3,0,0,0,114,1,0,0,0,114,131,0,0, + 0,114,121,0,0,0,114,33,0,0,0,41,6,114,42,0, + 0,0,114,123,0,0,0,114,142,0,0,0,114,143,0,0, + 0,114,176,0,0,0,114,122,0,0,0,41,3,218,7,103, + 108,111,98,97,108,115,114,88,0,0,0,114,170,0,0,0, + 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, + 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, + 95,95,4,4,0,0,115,22,0,0,0,0,7,15,1,12, + 1,7,1,15,1,12,1,9,2,13,1,10,1,12,1,19, + 1,114,198,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,0,5,0,0,0,67,0,0,0,115,227,0,0,0, + 124,4,0,100,1,0,107,2,0,114,27,0,116,0,0,124, + 0,0,131,1,0,125,5,0,110,54,0,124,1,0,100,2, + 0,107,9,0,114,45,0,124,1,0,110,3,0,105,0,0, + 125,6,0,116,1,0,124,6,0,131,1,0,125,7,0,116, + 0,0,124,0,0,124,7,0,124,4,0,131,3,0,125,5, + 0,124,3,0,115,207,0,124,4,0,100,1,0,107,2,0, + 114,122,0,116,0,0,124,0,0,106,2,0,100,3,0,131, + 1,0,100,1,0,25,131,1,0,83,124,0,0,115,132,0, + 124,5,0,83,116,3,0,124,0,0,131,1,0,116,3,0, + 124,0,0,106,2,0,100,3,0,131,1,0,100,1,0,25, + 131,1,0,24,125,8,0,116,4,0,106,5,0,124,5,0, + 106,6,0,100,2,0,116,3,0,124,5,0,106,6,0,131, + 1,0,124,8,0,24,133,2,0,25,25,83,110,16,0,116, + 7,0,124,5,0,124,3,0,116,0,0,131,3,0,83,100, + 2,0,83,41,4,97,214,1,0,0,73,109,112,111,114,116, + 32,97,32,109,111,100,117,108,101,46,10,10,32,32,32,32, + 84,104,101,32,39,103,108,111,98,97,108,115,39,32,97,114, + 103,117,109,101,110,116,32,105,115,32,117,115,101,100,32,116, + 111,32,105,110,102,101,114,32,119,104,101,114,101,32,116,104, + 101,32,105,109,112,111,114,116,32,105,115,32,111,99,99,117, + 114,105,110,103,32,102,114,111,109,10,32,32,32,32,116,111, + 32,104,97,110,100,108,101,32,114,101,108,97,116,105,118,101, + 32,105,109,112,111,114,116,115,46,32,84,104,101,32,39,108, + 111,99,97,108,115,39,32,97,114,103,117,109,101,110,116,32, + 105,115,32,105,103,110,111,114,101,100,46,32,84,104,101,10, + 32,32,32,32,39,102,114,111,109,108,105,115,116,39,32,97, + 114,103,117,109,101,110,116,32,115,112,101,99,105,102,105,101, + 115,32,119,104,97,116,32,115,104,111,117,108,100,32,101,120, + 105,115,116,32,97,115,32,97,116,116,114,105,98,117,116,101, + 115,32,111,110,32,116,104,101,32,109,111,100,117,108,101,10, + 32,32,32,32,98,101,105,110,103,32,105,109,112,111,114,116, + 101,100,32,40,101,46,103,46,32,96,96,102,114,111,109,32, + 109,111,100,117,108,101,32,105,109,112,111,114,116,32,60,102, + 114,111,109,108,105,115,116,62,96,96,41,46,32,32,84,104, + 101,32,39,108,101,118,101,108,39,10,32,32,32,32,97,114, + 103,117,109,101,110,116,32,114,101,112,114,101,115,101,110,116, + 115,32,116,104,101,32,112,97,99,107,97,103,101,32,108,111, + 99,97,116,105,111,110,32,116,111,32,105,109,112,111,114,116, + 32,102,114,111,109,32,105,110,32,97,32,114,101,108,97,116, + 105,118,101,10,32,32,32,32,105,109,112,111,114,116,32,40, + 101,46,103,46,32,96,96,102,114,111,109,32,46,46,112,107, + 103,32,105,109,112,111,114,116,32,109,111,100,96,96,32,119, + 111,117,108,100,32,104,97,118,101,32,97,32,39,108,101,118, + 101,108,39,32,111,102,32,50,41,46,10,10,32,32,32,32, + 114,33,0,0,0,78,114,121,0,0,0,41,8,114,187,0, + 0,0,114,198,0,0,0,218,9,112,97,114,116,105,116,105, + 111,110,114,168,0,0,0,114,14,0,0,0,114,21,0,0, + 0,114,1,0,0,0,114,195,0,0,0,41,9,114,15,0, + 0,0,114,197,0,0,0,218,6,108,111,99,97,108,115,114, + 193,0,0,0,114,171,0,0,0,114,89,0,0,0,90,8, + 103,108,111,98,97,108,115,95,114,170,0,0,0,90,7,99, + 117,116,95,111,102,102,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,10,95,95,105,109,112,111,114,116,95, + 95,25,4,0,0,115,26,0,0,0,0,11,12,1,15,2, + 24,1,12,1,18,1,6,3,12,1,23,1,6,1,4,4, + 35,3,40,2,114,201,0,0,0,99,1,0,0,0,0,0, + 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,53, + 0,0,0,116,0,0,106,1,0,124,0,0,131,1,0,125, + 1,0,124,1,0,100,0,0,107,8,0,114,43,0,116,2, + 0,100,1,0,124,0,0,23,131,1,0,130,1,0,116,3, + 0,124,1,0,131,1,0,83,41,2,78,122,25,110,111,32, + 98,117,105,108,116,45,105,110,32,109,111,100,117,108,101,32, + 110,97,109,101,100,32,41,4,114,151,0,0,0,114,155,0, + 0,0,114,77,0,0,0,114,150,0,0,0,41,2,114,15, + 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,18,95,98,117,105,108,116,105, + 110,95,102,114,111,109,95,110,97,109,101,60,4,0,0,115, + 8,0,0,0,0,1,15,1,12,1,16,1,114,202,0,0, + 0,99,2,0,0,0,0,0,0,0,12,0,0,0,12,0, + 0,0,67,0,0,0,115,74,1,0,0,124,1,0,97,0, + 0,124,0,0,97,1,0,116,2,0,116,1,0,131,1,0, + 125,2,0,120,123,0,116,1,0,106,3,0,106,4,0,131, + 0,0,68,93,106,0,92,2,0,125,3,0,125,4,0,116, + 5,0,124,4,0,124,2,0,131,2,0,114,40,0,124,3, + 0,116,1,0,106,6,0,107,6,0,114,91,0,116,7,0, + 125,5,0,110,27,0,116,0,0,106,8,0,124,3,0,131, + 1,0,114,40,0,116,9,0,125,5,0,110,3,0,113,40, + 0,116,10,0,124,4,0,124,5,0,131,2,0,125,6,0, + 116,11,0,124,6,0,124,4,0,131,2,0,1,113,40,0, + 87,116,1,0,106,3,0,116,12,0,25,125,7,0,120,73, + 0,100,5,0,68,93,65,0,125,8,0,124,8,0,116,1, + 0,106,3,0,107,7,0,114,206,0,116,13,0,124,8,0, + 131,1,0,125,9,0,110,13,0,116,1,0,106,3,0,124, + 8,0,25,125,9,0,116,14,0,124,7,0,124,8,0,124, + 9,0,131,3,0,1,113,170,0,87,121,16,0,116,13,0, + 100,2,0,131,1,0,125,10,0,87,110,24,0,4,116,15, + 0,107,10,0,114,25,1,1,1,1,100,3,0,125,10,0, + 89,110,1,0,88,116,14,0,124,7,0,100,2,0,124,10, + 0,131,3,0,1,116,13,0,100,4,0,131,1,0,125,11, + 0,116,14,0,124,7,0,100,4,0,124,11,0,131,3,0, + 1,100,3,0,83,41,6,122,250,83,101,116,117,112,32,105, + 109,112,111,114,116,108,105,98,32,98,121,32,105,109,112,111, + 114,116,105,110,103,32,110,101,101,100,101,100,32,98,117,105, + 108,116,45,105,110,32,109,111,100,117,108,101,115,32,97,110, + 100,32,105,110,106,101,99,116,105,110,103,32,116,104,101,109, + 10,32,32,32,32,105,110,116,111,32,116,104,101,32,103,108, + 111,98,97,108,32,110,97,109,101,115,112,97,99,101,46,10, + 10,32,32,32,32,65,115,32,115,121,115,32,105,115,32,110, + 101,101,100,101,100,32,102,111,114,32,115,121,115,46,109,111, + 100,117,108,101,115,32,97,99,99,101,115,115,32,97,110,100, + 32,95,105,109,112,32,105,115,32,110,101,101,100,101,100,32, + 116,111,32,108,111,97,100,32,98,117,105,108,116,45,105,110, + 10,32,32,32,32,109,111,100,117,108,101,115,44,32,116,104, + 111,115,101,32,116,119,111,32,109,111,100,117,108,101,115,32, + 109,117,115,116,32,98,101,32,101,120,112,108,105,99,105,116, + 108,121,32,112,97,115,115,101,100,32,105,110,46,10,10,32, + 32,32,32,114,142,0,0,0,114,34,0,0,0,78,114,62, + 0,0,0,41,1,122,9,95,119,97,114,110,105,110,103,115, + 41,16,114,57,0,0,0,114,14,0,0,0,114,13,0,0, + 0,114,21,0,0,0,218,5,105,116,101,109,115,114,178,0, + 0,0,114,76,0,0,0,114,151,0,0,0,114,82,0,0, + 0,114,161,0,0,0,114,132,0,0,0,114,137,0,0,0, + 114,1,0,0,0,114,202,0,0,0,114,5,0,0,0,114, + 77,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, + 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, + 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, + 0,114,89,0,0,0,114,99,0,0,0,114,88,0,0,0, + 90,11,115,101,108,102,95,109,111,100,117,108,101,90,12,98, + 117,105,108,116,105,110,95,110,97,109,101,90,14,98,117,105, + 108,116,105,110,95,109,111,100,117,108,101,90,13,116,104,114, + 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, + 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, + 112,67,4,0,0,115,50,0,0,0,0,9,6,1,6,3, + 12,1,28,1,15,1,15,1,9,1,15,1,9,2,3,1, + 15,1,17,3,13,1,13,1,15,1,15,2,13,1,20,3, + 3,1,16,1,13,2,11,1,16,3,12,1,114,206,0,0, + 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, + 0,0,67,0,0,0,115,87,0,0,0,116,0,0,124,0, + 0,124,1,0,131,2,0,1,116,1,0,106,2,0,106,3, + 0,116,4,0,131,1,0,1,116,1,0,106,2,0,106,3, + 0,116,5,0,131,1,0,1,100,1,0,100,2,0,108,6, + 0,125,2,0,124,2,0,97,7,0,124,2,0,106,8,0, + 116,1,0,106,9,0,116,10,0,25,131,1,0,1,100,2, + 0,83,41,3,122,50,73,110,115,116,97,108,108,32,105,109, + 112,111,114,116,108,105,98,32,97,115,32,116,104,101,32,105, + 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, + 32,105,109,112,111,114,116,46,114,33,0,0,0,78,41,11, + 114,206,0,0,0,114,14,0,0,0,114,175,0,0,0,114, + 113,0,0,0,114,151,0,0,0,114,161,0,0,0,218,26, + 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, + 98,95,101,120,116,101,114,110,97,108,114,119,0,0,0,218, + 8,95,105,110,115,116,97,108,108,114,21,0,0,0,114,1, + 0,0,0,41,3,114,204,0,0,0,114,205,0,0,0,114, + 207,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,114,208,0,0,0,114,4,0,0,115,12,0,0, + 0,0,2,13,2,16,1,16,3,12,1,6,1,114,208,0, + 0,0,41,51,114,3,0,0,0,114,119,0,0,0,114,12, + 0,0,0,114,16,0,0,0,114,17,0,0,0,114,59,0, + 0,0,114,41,0,0,0,114,48,0,0,0,114,31,0,0, + 0,114,32,0,0,0,114,53,0,0,0,114,54,0,0,0, + 114,56,0,0,0,114,63,0,0,0,114,65,0,0,0,114, + 75,0,0,0,114,81,0,0,0,114,84,0,0,0,114,90, + 0,0,0,114,101,0,0,0,114,102,0,0,0,114,106,0, + 0,0,114,85,0,0,0,218,6,111,98,106,101,99,116,90, + 9,95,80,79,80,85,76,65,84,69,114,132,0,0,0,114, + 137,0,0,0,114,145,0,0,0,114,97,0,0,0,114,86, + 0,0,0,114,149,0,0,0,114,150,0,0,0,114,87,0, + 0,0,114,151,0,0,0,114,161,0,0,0,114,166,0,0, + 0,114,172,0,0,0,114,174,0,0,0,114,177,0,0,0, + 114,182,0,0,0,114,192,0,0,0,114,183,0,0,0,114, + 185,0,0,0,114,186,0,0,0,114,187,0,0,0,114,195, + 0,0,0,114,198,0,0,0,114,201,0,0,0,114,202,0, + 0,0,114,206,0,0,0,114,208,0,0,0,114,10,0,0, + 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, + 218,8,60,109,111,100,117,108,101,62,8,0,0,0,115,96, + 0,0,0,6,17,6,2,12,8,12,4,19,20,6,2,6, + 3,22,4,19,68,19,21,19,19,12,19,12,19,12,11,18, + 8,12,11,12,12,12,16,12,36,19,27,19,101,24,26,9, + 3,18,45,18,60,12,18,12,17,12,25,12,29,12,23,12, + 16,19,73,19,77,19,13,12,9,12,9,15,40,12,17,6, + 1,10,2,12,27,12,6,18,24,12,32,12,21,24,35,12, + 7,12,47, }; -- cgit v0.12 From 60255b67b986d4c448153bef16755599afbfdaa2 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Fri, 15 Jan 2016 15:01:33 -0800 Subject: revert change 87a9dff5106c: pure Enum members again evaluate to True; update Finer Points section of docs to cover boolean evaluation; add more tests for pure and mixed boolean evaluation --- Doc/library/enum.rst | 11 ++++++++++- Lib/enum.py | 3 --- Lib/test/test_enum.py | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index a76f5a3..377ac3e 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -257,7 +257,7 @@ members are not integers (but see `IntEnum`_ below):: >>> Color.red < Color.blue Traceback (most recent call last): File "", line 1, in - TypeError: '<' not supported between instances of 'Color' and 'Color' + TypeError: unorderable types: Color() < Color() Equality comparisons are defined though:: @@ -747,6 +747,15 @@ besides the :class:`Enum` member you looking for:: .. versionchanged:: 3.5 +Boolean evaluation: Enum classes that are mixed with non-Enum types (such as +:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in +type's rules; otherwise, all members evaluate as ``True``. To make your own +Enum's boolean evaluation depend on the member's value add the following to +your class:: + + def __bool__(self): + return bool(self._value_) + The :attr:`__members__` attribute is only available on the class. If you give your :class:`Enum` subclass extra methods, like the `Planet`_ diff --git a/Lib/enum.py b/Lib/enum.py index 35a9c77..ac89d6b 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -482,9 +482,6 @@ class Enum(metaclass=EnumMeta): def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) - def __bool__(self): - return bool(self._value_) - def __dir__(self): added_behavior = [ m diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index e4e6c2b..7985948 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -272,11 +272,26 @@ class TestEnum(unittest.TestCase): _any_name_ = 9 def test_bool(self): + # plain Enum members are always True class Logic(Enum): true = True false = False self.assertTrue(Logic.true) - self.assertFalse(Logic.false) + self.assertTrue(Logic.false) + # unless overridden + class RealLogic(Enum): + true = True + false = False + def __bool__(self): + return bool(self._value_) + self.assertTrue(RealLogic.true) + self.assertFalse(RealLogic.false) + # mixed Enums depend on mixed-in type + class IntLogic(int, Enum): + true = 1 + false = 0 + self.assertTrue(IntLogic.true) + self.assertFalse(IntLogic.false) def test_contains(self): Season = self.Season -- cgit v0.12 From 7978e10441e84482d3d28e7c449d68b77c6310b6 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:26:54 +0000 Subject: Issue #23883: Missing fileinput.__all__ APIs; patch by Mauro SM Rodrigues --- Lib/fileinput.py | 3 ++- Lib/test/test_fileinput.py | 8 ++++++++ Misc/ACKS | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 021e39f..3543653 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -82,7 +82,8 @@ XXX Possible additions: import sys, os __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno", - "isfirstline", "isstdin", "FileInput"] + "fileno", "isfirstline", "isstdin", "FileInput", "hook_compressed", + "hook_encoded"] _state = None diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index 91c1166..ad81304 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -24,6 +24,7 @@ from fileinput import FileInput, hook_encoded from test.support import verbose, TESTFN, run_unittest, check_warnings from test.support import unlink as safe_unlink +from test import support from unittest import mock @@ -913,5 +914,12 @@ class Test_hook_encoded(unittest.TestCase): check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac']) +class MiscTest(unittest.TestCase): + + def test_all(self): + blacklist = {'DEFAULT_BUFSIZE'} + support.check__all__(self, fileinput, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index b243181..1df5d24 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1219,6 +1219,7 @@ Mark Roddy Kevin Rodgers Sean Rodman Giampaolo Rodola +Mauro S. M. Rodrigues Elson Rodriguez Adi Roiban Luis Rojas -- cgit v0.12 From 4eb376c441f99e9ea50beae5309020a1469393ac Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:49:30 +0000 Subject: Issue #23883: Add missing APIs to calendar.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Joel Taddei and Jacek KoÅ‚odziej. --- Lib/calendar.py | 4 +++- Lib/test/test_calendar.py | 9 +++++++++ Misc/ACKS | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Lib/calendar.py b/Lib/calendar.py index 196a075..85baf2e 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -12,7 +12,9 @@ import locale as _locale __all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday", "firstweekday", "isleap", "leapdays", "weekday", "monthrange", "monthcalendar", "prmonth", "month", "prcal", "calendar", - "timegm", "month_name", "month_abbr", "day_name", "day_abbr"] + "timegm", "month_name", "month_abbr", "day_name", "day_abbr", + "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", + "LocaleHTMLCalendar", "weekheader"] # Exception raised for bad input (with string parameter for details) error = ValueError diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d9d3128..d95ad04 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -815,5 +815,14 @@ class CommandLineTestCase(unittest.TestCase): b'href="custom.css" />', stdout) +class MiscTestCase(unittest.TestCase): + def test__all__(self): + blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', + 'SATURDAY', 'SUNDAY', 'different_locale', 'c', + 'prweek', 'week', 'format', 'formatstring', 'main'} + support.check__all__(self, calendar, blacklist=blacklist) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/ACKS b/Misc/ACKS index 1df5d24..ebc0d5d 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1419,6 +1419,7 @@ Péter Szabó John Szakmeister Amir Szekely Maciej Szulik +Joel Taddei Arfrever Frehtes Taifersar Arahesis Hideaki Takahashi Takase Arihiro -- cgit v0.12 From 104dcdab593b50596666b9f27fa6bfedc376deae Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 06:59:13 +0000 Subject: Issue #23883: Add missing APIs to tarfile.__all__ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Joel Taddei and Jacek KoÅ‚odziej. --- Lib/tarfile.py | 5 ++++- Lib/test/test_tarfile.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 80b6e35..7d43b1e 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -64,7 +64,10 @@ except NameError: pass # from tarfile import * -__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"] +__all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError", "ReadError", + "CompressionError", "StreamError", "ExtractError", "HeaderError", + "ENCODING", "USTAR_FORMAT", "GNU_FORMAT", "PAX_FORMAT", + "DEFAULT_FORMAT", "open"] #--------------------------------------------------------- # tar constants diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 1412cae..fee1a6c 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1997,6 +1997,24 @@ class MiscTest(unittest.TestCase): with self.assertRaises(ValueError): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) + def test__all__(self): + blacklist = {'version', 'bltn_open', 'symlink_exception', + 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', + 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', + 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', + 'SYMTYPE', 'CHRTYPE', 'BLKTYPE', 'DIRTYPE', 'FIFOTYPE', + 'CONTTYPE', 'GNUTYPE_LONGNAME', 'GNUTYPE_LONGLINK', + 'GNUTYPE_SPARSE', 'XHDTYPE', 'XGLTYPE', 'SOLARIS_XHDTYPE', + 'SUPPORTED_TYPES', 'REGULAR_TYPES', 'GNU_TYPES', + 'PAX_FIELDS', 'PAX_NAME_FIELDS', 'PAX_NUMBER_FIELDS', + 'stn', 'nts', 'nti', 'itn', 'calc_chksums', 'copyfileobj', + 'filemode', + 'EmptyHeaderError', 'TruncatedHeaderError', + 'EOFHeaderError', 'InvalidHeaderError', + 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'main'} + support.check__all__(self, tarfile, blacklist=blacklist) + class CommandLineTest(unittest.TestCase): -- cgit v0.12 From e8afd01db8c96ac686ab1834a04c0467e8d092e9 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 07:01:46 +0000 Subject: Issue #23883: Update news --- Doc/whatsnew/3.6.rst | 5 +++-- Misc/NEWS | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index a5b9be2..f54cf12 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -251,8 +251,9 @@ Changes in the Python API :exc:`PendingDeprecationWarning`. * The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: :mod:`csv`, :mod:`enum`, - :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`threading` and + attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, + :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, + :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. diff --git a/Misc/NEWS b/Misc/NEWS index f87703f..0e46c8d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -275,9 +275,10 @@ Library anything as they could potentially signal a different process. - Issue #23883: Added missing APIs to __all__ to match the documented APIs - for the following modules: csv, enum, ftplib, logging, optparse, threading - and wave. Also added a test.support.check__all__() helper. Patches by - Jacek KoÅ‚odziej. + for the following modules: calendar, csv, enum, fileinput, ftplib, logging, + optparse, tarfile, threading and wave. Also added a + test.support.check__all__() helper. Patches by Jacek KoÅ‚odziej, Mauro + S. M. Rodrigues and Joel Taddei. - Issue #25590: In the Readline completer, only call getattr() once per attribute. Also complete names of attributes such as properties and slots -- cgit v0.12 From 2c1d3e320fa26d2f8da3196a4966d2c9ece77430 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 16 Jan 2016 11:05:11 +0200 Subject: Issue #23883: Removed redundant names from blacklists. --- Lib/test/test_calendar.py | 2 +- Lib/test/test_tarfile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d95ad04..6dad058 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -817,7 +817,7 @@ class CommandLineTestCase(unittest.TestCase): class MiscTestCase(unittest.TestCase): def test__all__(self): - blacklist = {'error', 'mdays', 'January', 'February', 'EPOCH', + blacklist = {'mdays', 'January', 'February', 'EPOCH', 'MONDAY', 'TUESDAY', 'WEDNESDAY', 'THURSDAY', 'FRIDAY', 'SATURDAY', 'SUNDAY', 'different_locale', 'c', 'prweek', 'week', 'format', 'formatstring', 'main'} diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fee1a6c..9ffa2b5 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'bltn_open', 'symlink_exception', + blacklist = {'version', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', @@ -2011,7 +2011,7 @@ class MiscTest(unittest.TestCase): 'filemode', 'EmptyHeaderError', 'TruncatedHeaderError', 'EOFHeaderError', 'InvalidHeaderError', - 'SubsequentHeaderError', 'ExFileObject', 'TarIter', + 'SubsequentHeaderError', 'ExFileObject', 'main'} support.check__all__(self, tarfile, blacklist=blacklist) -- cgit v0.12 From 5318d1048f701f5bfdc21aeca3bd6bc876d304b0 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 16 Jan 2016 11:01:14 +0000 Subject: Issue #23883: grp and pwd are None on Windows --- Lib/test/test_tarfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 9ffa2b5..03875ab 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1998,7 +1998,7 @@ class MiscTest(unittest.TestCase): tarfile.itn(0x10000000000, 6, tarfile.GNU_FORMAT) def test__all__(self): - blacklist = {'version', 'symlink_exception', + blacklist = {'version', 'grp', 'pwd', 'symlink_exception', 'NUL', 'BLOCKSIZE', 'RECORDSIZE', 'GNU_MAGIC', 'POSIX_MAGIC', 'LENGTH_NAME', 'LENGTH_LINK', 'LENGTH_PREFIX', 'REGTYPE', 'AREGTYPE', 'LNKTYPE', -- cgit v0.12 From 2ae4ea54a23468695b1bab4aca1befc3681e8bfb Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Sat, 16 Jan 2016 12:39:53 -0800 Subject: use public 'value' --- Doc/library/enum.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 377ac3e..1fa6e73 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -754,7 +754,7 @@ Enum's boolean evaluation depend on the member's value add the following to your class:: def __bool__(self): - return bool(self._value_) + return bool(self.value) The :attr:`__members__` attribute is only available on the class. -- cgit v0.12 From 5f6ccc787e3d554bf4d759eb532826151809a8dc Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Sun, 17 Jan 2016 12:28:43 +0100 Subject: Issue #26139: libmpdec: disable /W4 warning (non-standard dllimport behavior). --- Modules/_decimal/libmpdec/memory.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/_decimal/libmpdec/memory.c b/Modules/_decimal/libmpdec/memory.c index 0f41fe5..61eb633 100644 --- a/Modules/_decimal/libmpdec/memory.c +++ b/Modules/_decimal/libmpdec/memory.c @@ -33,6 +33,11 @@ #include "memory.h" +#if defined(_MSC_VER) + #pragma warning(disable : 4232) +#endif + + /* Guaranteed minimum allocation for a coefficient. May be changed once at program start using mpd_setminalloc(). */ mpd_ssize_t MPD_MINALLOC = MPD_MINALLOC_MIN; -- cgit v0.12 From 613065b60d823efa6e2be0e0e53079a44305b6b2 Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 17 Jan 2016 20:12:16 -0800 Subject: Issue26069 - Update whatsnew/3.6.rst on traceback module's api removals. --- Doc/whatsnew/3.6.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index f54cf12..0e15eda 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -227,6 +227,13 @@ API and Feature Removals :func:`inspect.getmodulename` should be used for obtaining the module name for a given path. +* ``traceback.Ignore`` class and ``traceback.usage``, ``traceback.modname``, + ``traceback.fullmodname``, ``traceback.find_lines_from_code``, + ``traceback.find_lines``, ``traceback.find_strings``, + ``traceback.find_executable_lines`` methods were removed from the + :mod:`traceback` module. They were undocumented methods deprecated since + Python 3.2 and equivalent functionality is available from private methods. + Porting to Python 3.6 ===================== -- cgit v0.12 From 5c60ea3fa1e40c6a3a2e0e9da71f55f8d7d6b2b1 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 18 Jan 2016 07:53:59 +0100 Subject: Fix two instances of wrong indentation. --- Python/ast.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index 33f2597..a5d8dba 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3214,14 +3214,14 @@ ast_for_import_stmt(struct compiling *c, const node *n) alias_ty import_alias = alias_for_import_name(c, n, 1); if (!import_alias) return NULL; - asdl_seq_SET(aliases, 0, import_alias); + asdl_seq_SET(aliases, 0, import_alias); } else { for (i = 0; i < NCH(n); i += 2) { alias_ty import_alias = alias_for_import_name(c, CHILD(n, i), 1); if (!import_alias) return NULL; - asdl_seq_SET(aliases, i / 2, import_alias); + asdl_seq_SET(aliases, i / 2, import_alias); } } if (mod != NULL) -- cgit v0.12 From 4b5b06203e4dbe24579101e5d889080c43854453 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 18 Jan 2016 08:00:15 +0100 Subject: Fix indentation of continuation lines. --- Modules/_testcapimodule.c | 4 ++-- Python/pylifecycle.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index cc3f52d..fbb4aa4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1047,7 +1047,7 @@ test_k_code(PyObject *self) value = PyLong_AsUnsignedLongMask(num); if (value != ULONG_MAX) return raiseTestError("test_k_code", - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); + "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); PyTuple_SET_ITEM(tuple, 0, num); @@ -1066,7 +1066,7 @@ test_k_code(PyObject *self) value = PyLong_AsUnsignedLongMask(num); if (value != (unsigned long)-0x42) return raiseTestError("test_k_code", - "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); + "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); PyTuple_SET_ITEM(tuple, 0, num); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c84c46a..e9db7f6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -796,7 +796,7 @@ Py_NewInterpreter(void) if (initstdio() < 0) Py_FatalError( - "Py_Initialize: can't initialize sys standard streams"); + "Py_Initialize: can't initialize sys standard streams"); initmain(interp); if (!Py_NoSiteFlag) initsite(); -- cgit v0.12 From c437d0cb4e99bd58ff0150414b5d5f0b26605687 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 11:25:50 +0100 Subject: Fix test_compilepath() of test_compileall Issue #26101: Exclude Lib/test/ from sys.path in test_compilepath(). The directory contains invalid Python files like Lib/test/badsyntax_pep3120.py, whereas the test ensures that all files can be compiled. --- Lib/test/test_compileall.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 439c125..6d8a863 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -103,6 +103,18 @@ class CompileallTests(unittest.TestCase): force=False, quiet=2)) def test_compile_path(self): + # Exclude Lib/test/ which contains invalid Python files like + # Lib/test/badsyntax_pep3120.py + testdir = os.path.realpath(os.path.dirname(__file__)) + if testdir in sys.path: + self.addCleanup(setattr, sys, 'path', sys.path) + + sys.path = list(sys.path) + try: + sys.path.remove(testdir) + except ValueError: + pass + self.assertTrue(compileall.compile_path(quiet=2)) with test.test_importlib.util.import_state(path=[self.directory]): -- cgit v0.12 From 9def2843873edde3feec6eaf2ee60c4e48172164 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 18 Jan 2016 12:15:08 +0100 Subject: subprocess._optim_args_from_interpreter_flags() Issue #26100: * Add subprocess._optim_args_from_interpreter_flags() * Add test.support.optim_args_from_interpreter_flags() * Use new functions in distutils, test_cmd_line_script, test_compileall and test_inspect The change enables test_details() test of test_inspect when -O or -OO command line option is used. --- Lib/distutils/util.py | 15 +++++++++------ Lib/subprocess.py | 14 ++++++++++++-- Lib/test/support/__init__.py | 5 +++++ Lib/test/test_cmd_line_script.py | 5 ++--- Lib/test/test_compileall.py | 7 +++---- Lib/test/test_inspect.py | 7 ++++--- 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index e423325..fdcf6fa 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -7,8 +7,8 @@ one of the other *util.py modules. import os import re import importlib.util -import sys import string +import sys from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -350,6 +350,11 @@ def byte_compile (py_files, generated in indirect mode; unless you know what you're doing, leave it set to None. """ + + # Late import to fix a bootstrap issue: _posixsubprocess is built by + # setup.py, but setup.py uses distutils. + import subprocess + # nothing is done if sys.dont_write_bytecode is True if sys.dont_write_bytecode: raise DistutilsByteCompileError('byte-compiling is disabled.') @@ -412,11 +417,9 @@ byte_compile(files, optimize=%r, force=%r, script.close() - cmd = [sys.executable, script_name] - if optimize == 1: - cmd.insert(1, "-O") - elif optimize == 2: - cmd.insert(1, "-OO") + cmd = [sys.executable] + cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.append(script_name) spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index d1324b8..640519d 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -520,6 +520,16 @@ DEVNULL = -3 # but it's here so that it can be imported when Python is compiled without # threads. +def _optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + args = [] + value = sys.flags.optimize + if value > 0: + args.append('-' + 'O' * value) + return args + + def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current settings in sys.flags and sys.warnoptions.""" @@ -527,7 +537,6 @@ def _args_from_interpreter_flags(): 'debug': 'd', # 'inspect': 'i', # 'interactive': 'i', - 'optimize': 'O', 'dont_write_bytecode': 'B', 'no_user_site': 's', 'no_site': 'S', @@ -535,8 +544,9 @@ def _args_from_interpreter_flags(): 'verbose': 'v', 'bytes_warning': 'b', 'quiet': 'q', + # -O is handled in _optim_args_from_interpreter_flags() } - args = [] + args = _optim_args_from_interpreter_flags() for flag, opt in flag_opt_map.items(): v = getattr(sys.flags, flag) if v > 0: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2969b36..58dcc8a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2053,6 +2053,11 @@ def args_from_interpreter_flags(): settings in sys.flags and sys.warnoptions.""" return subprocess._args_from_interpreter_flags() +def optim_args_from_interpreter_flags(): + """Return a list of command-line arguments reproducing the current + optimization settings in sys.flags.""" + return subprocess._optim_args_from_interpreter_flags() + #============================================================ # Support for assertions about logging. #============================================================ diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index afac62a..9398040 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -138,9 +138,8 @@ class CmdLineTest(unittest.TestCase): expected_argv0, expected_path0, expected_package, expected_loader, *cmd_line_switches): - if not __debug__: - cmd_line_switches += ('-' + 'O' * sys.flags.optimize,) - run_args = cmd_line_switches + (script_name,) + tuple(example_args) + run_args = [*support.optim_args_from_interpreter_flags(), + *cmd_line_switches, script_name, *example_args] rc, out, err = assert_python_ok(*run_args, __isolated=False) self._check_output(script_name, rc, out + err, expected_file, expected_argv0, expected_path0, diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 6d8a863..9b424a7 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -231,10 +231,9 @@ class CommandLineTests(unittest.TestCase): raise unittest.SkipTest('not all entries on sys.path are writable') def _get_run_args(self, args): - interp_args = ['-S'] - if sys.flags.optimize: - interp_args.append({1 : '-O', 2 : '-OO'}[sys.flags.optimize]) - return interp_args + ['-m', 'compileall'] + list(args) + return [*support.optim_args_from_interpreter_flags(), + '-S', '-m', 'compileall', + *args] def assertRunOK(self, *args, **env_vars): rc, out, err = script_helper.assert_python_ok( diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 283c922..422a3d6 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -30,6 +30,7 @@ from test.support import MISSING_C_DOCSTRINGS, cpython_only from test.support.script_helper import assert_python_ok, assert_python_failure from test import inspect_fodder as mod from test import inspect_fodder2 as mod2 +from test import support from test.test_import import _ready_to_import @@ -3536,14 +3537,14 @@ class TestMain(unittest.TestCase): def test_details(self): module = importlib.import_module('unittest') - rc, out, err = assert_python_ok('-m', 'inspect', + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', 'unittest', '--details') output = out.decode() # Just a quick sanity check on the output self.assertIn(module.__name__, output) self.assertIn(module.__file__, output) - if not sys.flags.optimize: - self.assertIn(module.__cached__, output) + self.assertIn(module.__cached__, output) self.assertEqual(err, b'') -- cgit v0.12 From 9cc4ed5c7a86b2aa58176222107dbb01d40a3680 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 18 Jan 2016 18:49:57 +0200 Subject: Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). --- Doc/library/grp.rst | 3 +++ Lib/test/test_grp.py | 10 ++++++++++ Misc/NEWS | 2 ++ Modules/grpmodule.c | 21 ++++++++++++++++----- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index 8882140..c840cfe 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -42,6 +42,9 @@ It defines the following items: Return the group database entry for the given numeric group ID. :exc:`KeyError` is raised if the entry asked for cannot be found. + .. deprecated:: 3.6 + Since Python 3.6 the support of non-integer arguments like floats or + strings in :func:`getgrgid` is deprecated. .. function:: getgrnam(name) diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index 272b086..69095a3 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -92,5 +92,15 @@ class GroupDatabaseTestCase(unittest.TestCase): self.assertRaises(KeyError, grp.getgrgid, fakegid) + def test_noninteger_gid(self): + entries = grp.getgrall() + if not entries: + self.skipTest('no groups') + # Choose an existent gid. + gid = entries[0][2] + self.assertWarns(DeprecationWarning, grp.getgrgid, float(gid)) + self.assertWarns(DeprecationWarning, grp.getgrgid, str(gid)) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 4564e29..d3edac7 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -131,6 +131,8 @@ Core and Builtins Library ------- +- Issue #26129: Deprecated accepting non-integers in grp.getgrgid(). + - Issue #25850: Use cross-compilation by default for 64-bit Windows. - Issue #25822: Add docstrings to the fields of urllib.parse results. diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 403e434..5ad87f1 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -100,14 +100,25 @@ grp_getgrgid_impl(PyModuleDef *module, PyObject *id) gid_t gid; struct group *p; - py_int_id = PyNumber_Long(id); - if (!py_int_id) + if (!_Py_Gid_Converter(id, &gid)) { + if (!PyErr_ExceptionMatches(PyExc_TypeError)) { return NULL; - if (!_Py_Gid_Converter(py_int_id, &gid)) { + } + PyErr_Clear(); + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "group id must be int, not %.200", + id->ob_type->tp_name) < 0) { + return NULL; + } + py_int_id = PyNumber_Long(id); + if (!py_int_id) + return NULL; + if (!_Py_Gid_Converter(py_int_id, &gid)) { + Py_DECREF(py_int_id); + return NULL; + } Py_DECREF(py_int_id); - return NULL; } - Py_DECREF(py_int_id); if ((p = getgrgid(gid)) == NULL) { PyObject *gid_obj = _PyLong_FromGid(gid); -- cgit v0.12 From 31a858cbf1eca34f04dd425fa7f8d6f031e5de66 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 19 Jan 2016 14:09:33 +0200 Subject: Issue #16620: Got rid of using undocumented function glob.glob1(). --- Lib/msilib/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/msilib/__init__.py b/Lib/msilib/__init__.py index 6d0a28f..823644a 100644 --- a/Lib/msilib/__init__.py +++ b/Lib/msilib/__init__.py @@ -1,7 +1,7 @@ # Copyright (C) 2005 Martin v. Löwis # Licensed to PSF under a Contributor Agreement. from _msi import * -import glob +import fnmatch import os import re import string @@ -379,7 +379,13 @@ class Directory: def glob(self, pattern, exclude = None): """Add a list of files to the current component as specified in the glob pattern. Individual files can be excluded in the exclude list.""" - files = glob.glob1(self.absolute, pattern) + try: + files = os.listdir(self.absolute) + except OSError: + return [] + if pattern[:1] != '.': + files = (f for f in files if f[0] != '.') + files = fnmatch.filter(files, pattern) for f in files: if exclude and f in exclude: continue self.add_file(f) -- cgit v0.12 From f3914eb16d28ad9eb20fe5133d9aa83658bcc27f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Jan 2016 12:16:21 +0100 Subject: co_lnotab supports negative line number delta Issue #26107: The format of the co_lnotab attribute of code objects changes to support negative line number delta. Changes: * assemble_lnotab(): if line number delta is less than -128 or greater than 127, emit multiple (offset_delta, lineno_delta) in co_lnotab * update functions decoding co_lnotab to use signed 8-bit integers - dis.findlinestarts() - PyCode_Addr2Line() - _PyCode_CheckLineNumber() - frame_setlineno() * update lnotab_notes.txt * increase importlib MAGIC_NUMBER to 3361 * document the change in What's New in Python 3.6 * cleanup also PyCode_Optimize() to use better variable names --- Doc/whatsnew/3.6.rst | 10 ++ Include/code.h | 2 +- Lib/dis.py | 7 +- Lib/importlib/_bootstrap_external.py | 5 +- Misc/NEWS | 3 + Objects/codeobject.c | 11 +- Objects/frameobject.c | 2 +- Objects/lnotab_notes.txt | 37 +++--- Python/compile.c | 25 +++-- Python/importlib_external.h | 211 ++++++++++++++++++----------------- Python/peephole.c | 51 +++++---- 11 files changed, 203 insertions(+), 161 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0e15eda..8064537 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -244,6 +244,16 @@ that may require changes to your code. Changes in the Python API ------------------------- +* The format of the ``co_lnotab`` attribute of code objects changed to support + negative line number delta. By default, Python does not emit bytecode with + negative line number delta. Functions using ``frame.f_lineno``, + ``PyFrame_GetLineNumber()`` or ``PyCode_Addr2Line()`` are not affected. + Functions decoding directly ``co_lnotab`` should be updated to use a signed + 8-bit integer type for the line number delta, but it's only required to + support applications using negative line number delta. See + ``Objects/lnotab_notes.txt`` for the ``co_lnotab`` format and how to decode + it, and see the :pep:`511` for the rationale. + * The functions in the :mod:`compileall` module now return booleans instead of ``1`` or ``0`` to represent success or failure, respectively. Thanks to booleans being a subclass of integers, this should only be an issue if you diff --git a/Include/code.h b/Include/code.h index 56e6ec1..3ce2084 100644 --- a/Include/code.h +++ b/Include/code.h @@ -117,7 +117,7 @@ PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, #endif PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, - PyObject *names, PyObject *lineno_obj); + PyObject *names, PyObject *lnotab); #ifdef __cplusplus } diff --git a/Lib/dis.py b/Lib/dis.py index 3540b47..2e80e17 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -397,8 +397,8 @@ def findlinestarts(code): Generate pairs (offset, lineno) as described in Python/compile.c. """ - byte_increments = list(code.co_lnotab[0::2]) - line_increments = list(code.co_lnotab[1::2]) + byte_increments = code.co_lnotab[0::2] + line_increments = code.co_lnotab[1::2] lastlineno = None lineno = code.co_firstlineno @@ -409,6 +409,9 @@ def findlinestarts(code): yield (addr, lineno) lastlineno = lineno addr += byte_incr + if line_incr >= 0x80: + # line_increments is an array of 8-bit signed integers + line_incr -= 0x100 lineno += line_incr if lineno != lastlineno: yield (addr, lineno) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f274501..71098f1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -223,13 +223,14 @@ _code_type = type(_write_atomic.__code__) # Python 3.5b1 3330 (PEP 448: Additional Unpacking Generalizations) # Python 3.5b2 3340 (fix dictionary display evaluation order #11205) # Python 3.5b2 3350 (add GET_YIELD_FROM_ITER opcode #24400) -# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483) +# Python 3.6a0 3360 (add FORMAT_VALUE opcode #25483 +# Python 3.6a0 3361 (lineno delta of code.co_lnotab becomes signed) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually # due to the addition of new opcodes). -MAGIC_NUMBER = (3360).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3361).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Misc/NEWS b/Misc/NEWS index 46464de..8c6864c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #26107: The format of the ``co_lnotab`` attribute of code objects + changes to support negative line number delta. + - Issue #26154: Add a new private _PyThreadState_UncheckedGet() function to get the current Python thread state, but don't issue a fatal error if it is NULL. This new function must be used instead of accessing directly the diff --git a/Objects/codeobject.c b/Objects/codeobject.c index b0e3446..0f03dfe 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -557,7 +557,8 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) addr += *p++; if (addr > addrq) break; - line += *p++; + line += (signed char)*p; + p++; } return line; } @@ -592,17 +593,19 @@ _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds) if (addr + *p > lasti) break; addr += *p++; - if (*p) + if ((signed char)*p) bounds->ap_lower = addr; - line += *p++; + line += (signed char)*p; + p++; --size; } if (size > 0) { while (--size >= 0) { addr += *p++; - if (*p++) + if ((signed char)*p) break; + p++; } bounds->ap_upper = addr; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 425a9ee..da2d2ed 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -137,7 +137,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno) new_lasti = -1; for (offset = 0; offset < lnotab_len; offset += 2) { addr += lnotab[offset]; - line += lnotab[offset+1]; + line += (signed char)lnotab[offset+1]; if (line >= new_lineno) { new_lasti = addr; new_lineno = line; diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index d247edd..8af10db 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -12,42 +12,47 @@ pairs. The details are important and delicate, best illustrated by example: 0 1 6 2 50 7 - 350 307 - 361 308 + 350 207 + 361 208 Instead of storing these numbers literally, we compress the list by storing only -the increments from one row to the next. Conceptually, the stored list might +the difference from one row to the next. Conceptually, the stored list might look like: - 0, 1, 6, 1, 44, 5, 300, 300, 11, 1 + 0, 1, 6, 1, 44, 5, 300, 200, 11, 1 -The above doesn't really work, but it's a start. Note that an unsigned byte -can't hold negative values, or values larger than 255, and the above example -contains two such values. So we make two tweaks: +The above doesn't really work, but it's a start. An unsigned byte (byte code +offset)) can't hold negative values, or values larger than 255, a signed byte +(line number) can't hold values larger than 127 or less than -128, and the +above example contains two such values. So we make two tweaks: - (a) there's a deep assumption that byte code offsets and their corresponding - line #s both increase monotonically, and - (b) if at least one column jumps by more than 255 from one row to the next, - more than one pair is written to the table. In case #b, there's no way to know - from looking at the table later how many were written. That's the delicate - part. A user of co_lnotab desiring to find the source line number - corresponding to a bytecode address A should do something like this + (a) there's a deep assumption that byte code offsets increase monotonically, + and + (b) if byte code offset jumps by more than 255 from one row to the next, or if + source code line number jumps by more than 127 or less than -128 from one row + to the next, more than one pair is written to the table. In case #b, + there's no way to know from looking at the table later how many were written. + That's the delicate part. A user of co_lnotab desiring to find the source + line number corresponding to a bytecode address A should do something like + this: lineno = addr = 0 for addr_incr, line_incr in co_lnotab: addr += addr_incr if addr > A: return lineno + if line_incr >= 0x80: + line_incr -= 0x100 lineno += line_incr (In C, this is implemented by PyCode_Addr2Line().) In order for this to work, when the addr field increments by more than 255, the line # increment in each pair generated must be 0 until the remaining addr increment is < 256. So, in the example above, assemble_lnotab in compile.c should not (as was actually done -until 2.2) expand 300, 300 to +until 2.2) expand 300, 200 to 255, 255, 45, 45, but to - 255, 0, 45, 255, 0, 45. + 255, 0, 45, 128, 0, 72. The above is sufficient to reconstruct line numbers for tracebacks, but not for line tracing. Tracing is handled by PyCode_CheckLineNumber() in codeobject.c diff --git a/Python/compile.c b/Python/compile.c index 0f619c4..f362ac6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4452,7 +4452,6 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_lineno = i->i_lineno - a->a_lineno; assert(d_bytecode >= 0); - assert(d_lineno >= 0); if(d_bytecode == 0 && d_lineno == 0) return 1; @@ -4482,9 +4481,21 @@ assemble_lnotab(struct assembler *a, struct instr *i) d_bytecode -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } - assert(d_bytecode <= 255); - if (d_lineno > 255) { - int j, nbytes, ncodes = d_lineno / 255; + assert(0 <= d_bytecode && d_bytecode <= 255); + + if (d_lineno < -128 || 127 < d_lineno) { + int j, nbytes, ncodes, k; + if (d_lineno < 0) { + k = -128; + /* use division on positive numbers */ + ncodes = (-d_lineno) / 128; + } + else { + k = 127; + ncodes = d_lineno / 127; + } + d_lineno -= ncodes * k; + assert(ncodes >= 1); nbytes = a->a_lnotab_off + 2 * ncodes; len = PyBytes_GET_SIZE(a->a_lnotab); if (nbytes >= len) { @@ -4502,15 +4513,15 @@ assemble_lnotab(struct assembler *a, struct instr *i) lnotab = (unsigned char *) PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off; *lnotab++ = d_bytecode; - *lnotab++ = 255; + *lnotab++ = k; d_bytecode = 0; for (j = 1; j < ncodes; j++) { *lnotab++ = 0; - *lnotab++ = 255; + *lnotab++ = k; } - d_lineno -= ncodes * 255; a->a_lnotab_off += ncodes * 2; } + assert(-128 <= d_lineno && d_lineno <= 127); len = PyBytes_GET_SIZE(a->a_lnotab); if (a->a_lnotab_off + 2 >= len) { diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 5828ceb..97c4948 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -258,7 +258,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,5,0,0,0,218,13,95,119,114,105,116,101,95,97,116, 111,109,105,99,99,0,0,0,115,26,0,0,0,0,5,24, 1,9,1,33,1,3,3,21,1,20,1,20,1,13,1,3, - 1,17,1,13,1,5,1,114,55,0,0,0,105,32,13,0, + 1,17,1,13,1,5,1,114,55,0,0,0,105,33,13,0, 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, @@ -368,7 +368,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,90,15,97,108,109,111,115,116,95,102,105,108,101,110,97, 109,101,114,4,0,0,0,114,4,0,0,0,114,5,0,0, 0,218,17,99,97,99,104,101,95,102,114,111,109,95,115,111, - 117,114,99,101,244,0,0,0,115,46,0,0,0,0,18,12, + 117,114,99,101,245,0,0,0,115,46,0,0,0,0,18,12, 1,9,1,7,1,12,1,6,1,12,1,18,1,18,1,24, 1,12,1,12,1,12,1,36,1,12,1,18,1,9,2,12, 1,12,1,12,1,12,1,21,1,21,1,114,79,0,0,0, @@ -447,7 +447,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, - 99,97,99,104,101,32,1,0,0,115,44,0,0,0,0,9, + 99,97,99,104,101,33,1,0,0,115,44,0,0,0,0,9, 18,1,12,1,18,1,18,1,12,1,9,1,15,1,15,1, 12,1,9,1,15,1,12,1,22,1,15,1,9,1,12,1, 22,1,12,1,9,1,12,1,19,1,114,85,0,0,0,99, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,36,0,0,0,90,9,101,120,116,101,110,115,105, 111,110,218,11,115,111,117,114,99,101,95,112,97,116,104,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,15, - 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,65, + 95,103,101,116,95,115,111,117,114,99,101,102,105,108,101,66, 1,0,0,115,20,0,0,0,0,7,18,1,4,1,24,1, 35,1,4,1,3,1,16,1,19,1,21,1,114,91,0,0, 0,99,1,0,0,0,0,0,0,0,1,0,0,0,11,0, @@ -499,7 +499,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,79,0,0,0,114,66,0,0,0,114,74,0,0, 0,41,1,218,8,102,105,108,101,110,97,109,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,11,95,103, - 101,116,95,99,97,99,104,101,100,84,1,0,0,115,16,0, + 101,116,95,99,97,99,104,101,100,85,1,0,0,115,16,0, 0,0,0,1,21,1,3,1,14,1,13,1,8,1,21,1, 4,2,114,95,0,0,0,99,1,0,0,0,0,0,0,0, 2,0,0,0,11,0,0,0,67,0,0,0,115,60,0,0, @@ -514,7 +514,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,114,41,0,0,0,114,40,0,0,0,41, 2,114,35,0,0,0,114,42,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,10,95,99,97,108, - 99,95,109,111,100,101,96,1,0,0,115,12,0,0,0,0, + 99,95,109,111,100,101,97,1,0,0,115,12,0,0,0,0, 2,3,1,19,1,13,1,11,3,10,1,114,97,0,0,0, 99,1,0,0,0,0,0,0,0,3,0,0,0,11,0,0, 0,3,0,0,0,115,84,0,0,0,100,1,0,135,0,0, @@ -554,7 +554,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,115,90,6,107,119,97,114,103,115,41,1,218,6,109,101, 116,104,111,100,114,4,0,0,0,114,5,0,0,0,218,19, 95,99,104,101,99,107,95,110,97,109,101,95,119,114,97,112, - 112,101,114,116,1,0,0,115,12,0,0,0,0,1,12,1, + 112,101,114,117,1,0,0,115,12,0,0,0,0,1,12,1, 12,1,15,1,6,1,25,1,122,40,95,99,104,101,99,107, 95,110,97,109,101,46,60,108,111,99,97,108,115,62,46,95, 99,104,101,99,107,95,110,97,109,101,95,119,114,97,112,112, @@ -573,7 +573,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,97,116,116,114,218,8,95,95,100,105,99,116,95,95,218, 6,117,112,100,97,116,101,41,3,90,3,110,101,119,90,3, 111,108,100,114,52,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,218,5,95,119,114,97,112,127,1, + 0,0,114,5,0,0,0,218,5,95,119,114,97,112,128,1, 0,0,115,8,0,0,0,0,1,25,1,15,1,29,1,122, 26,95,99,104,101,99,107,95,110,97,109,101,46,60,108,111, 99,97,108,115,62,46,95,119,114,97,112,41,3,218,10,95, @@ -581,7 +581,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,97,109,101,69,114,114,111,114,41,3,114,102,0,0,0, 114,103,0,0,0,114,113,0,0,0,114,4,0,0,0,41, 1,114,102,0,0,0,114,5,0,0,0,218,11,95,99,104, - 101,99,107,95,110,97,109,101,108,1,0,0,115,14,0,0, + 101,99,107,95,110,97,109,101,109,1,0,0,115,14,0,0, 0,0,8,21,7,3,1,13,1,13,2,17,5,13,1,114, 116,0,0,0,99,2,0,0,0,0,0,0,0,5,0,0, 0,4,0,0,0,67,0,0,0,115,84,0,0,0,124,0, @@ -611,7 +611,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,112,111,114,116,105,111,110,115,218,3,109,115,103,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,17, 95,102,105,110,100,95,109,111,100,117,108,101,95,115,104,105, - 109,136,1,0,0,115,10,0,0,0,0,10,21,1,24,1, + 109,137,1,0,0,115,10,0,0,0,0,10,21,1,24,1, 6,1,29,1,114,123,0,0,0,99,4,0,0,0,0,0, 0,0,11,0,0,0,19,0,0,0,67,0,0,0,115,252, 1,0,0,105,0,0,125,4,0,124,2,0,100,1,0,107, @@ -698,7 +698,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,11,115,111,117,114,99,101,95,115,105,122,101,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,25,95,118, 97,108,105,100,97,116,101,95,98,121,116,101,99,111,100,101, - 95,104,101,97,100,101,114,153,1,0,0,115,76,0,0,0, + 95,104,101,97,100,101,114,154,1,0,0,115,76,0,0,0, 0,11,6,1,12,1,13,3,6,1,12,1,10,1,16,1, 16,1,16,1,12,1,18,1,16,1,18,1,18,1,15,1, 16,1,15,1,18,1,15,1,16,1,12,1,12,1,3,1, @@ -729,7 +729,7 @@ const unsigned char _Py_M__importlib_external[] = { 5,114,53,0,0,0,114,98,0,0,0,114,89,0,0,0, 114,90,0,0,0,218,4,99,111,100,101,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,109, - 112,105,108,101,95,98,121,116,101,99,111,100,101,208,1,0, + 112,105,108,101,95,98,121,116,101,99,111,100,101,209,1,0, 0,115,16,0,0,0,0,2,15,1,15,1,16,1,12,1, 16,1,4,2,18,1,114,141,0,0,0,114,59,0,0,0, 99,3,0,0,0,0,0,0,0,4,0,0,0,3,0,0, @@ -749,7 +749,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,117,109,112,115,41,4,114,140,0,0,0,114,126,0,0, 0,114,134,0,0,0,114,53,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,17,95,99,111,100, - 101,95,116,111,95,98,121,116,101,99,111,100,101,220,1,0, + 101,95,116,111,95,98,121,116,101,99,111,100,101,221,1,0, 0,115,10,0,0,0,0,3,12,1,19,1,19,1,22,1, 114,144,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,4,0,0,0,67,0,0,0,115,89,0,0,0,100, @@ -778,7 +778,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,101,110,99,111,100,105,110,103,90,15,110,101,119,108, 105,110,101,95,100,101,99,111,100,101,114,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,13,100,101,99,111, - 100,101,95,115,111,117,114,99,101,230,1,0,0,115,10,0, + 100,101,95,115,111,117,114,99,101,231,1,0,0,115,10,0, 0,0,0,5,12,1,18,1,15,1,18,1,114,149,0,0, 0,114,120,0,0,0,218,26,115,117,98,109,111,100,117,108, 101,95,115,101,97,114,99,104,95,108,111,99,97,116,105,111, @@ -843,7 +843,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,105,120,101,115,114,153,0,0,0,90,7,100,105,114,110, 97,109,101,114,4,0,0,0,114,4,0,0,0,114,5,0, 0,0,218,23,115,112,101,99,95,102,114,111,109,95,102,105, - 108,101,95,108,111,99,97,116,105,111,110,247,1,0,0,115, + 108,101,95,108,111,99,97,116,105,111,110,248,1,0,0,115, 60,0,0,0,0,12,12,4,6,1,15,2,3,1,19,1, 13,1,5,8,24,1,9,3,12,1,22,1,21,1,15,1, 9,1,5,2,4,3,12,2,15,1,3,1,19,1,13,1, @@ -883,7 +883,7 @@ const unsigned char _Py_M__importlib_external[] = { 72,75,69,89,95,76,79,67,65,76,95,77,65,67,72,73, 78,69,41,2,218,3,99,108,115,218,3,107,101,121,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,14,95, - 111,112,101,110,95,114,101,103,105,115,116,114,121,69,2,0, + 111,112,101,110,95,114,101,103,105,115,116,114,121,70,2,0, 0,115,8,0,0,0,0,2,3,1,23,1,13,1,122,36, 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, @@ -910,7 +910,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,95,107,101,121,114,165,0,0,0,90,4,104,107,101,121, 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, - 99,104,95,114,101,103,105,115,116,114,121,76,2,0,0,115, + 99,104,95,114,101,103,105,115,116,114,121,77,2,0,0,115, 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, @@ -935,7 +935,7 @@ const unsigned char _Py_M__importlib_external[] = { 103,101,116,114,171,0,0,0,114,120,0,0,0,114,160,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,9,102,105,110,100,95,115,112,101, - 99,91,2,0,0,115,26,0,0,0,0,2,15,1,12,1, + 99,92,2,0,0,115,26,0,0,0,0,2,15,1,12,1, 4,1,3,1,14,1,13,1,9,1,22,1,21,1,9,1, 15,1,9,1,122,31,87,105,110,100,111,119,115,82,101,103, 105,115,116,114,121,70,105,110,100,101,114,46,102,105,110,100, @@ -954,7 +954,7 @@ const unsigned char _Py_M__importlib_external[] = { 175,0,0,0,114,120,0,0,0,41,4,114,164,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,158,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,11, - 102,105,110,100,95,109,111,100,117,108,101,107,2,0,0,115, + 102,105,110,100,95,109,111,100,117,108,101,108,2,0,0,115, 8,0,0,0,0,7,18,1,12,1,7,2,122,33,87,105, 110,100,111,119,115,82,101,103,105,115,116,114,121,70,105,110, 100,101,114,46,102,105,110,100,95,109,111,100,117,108,101,41, @@ -963,7 +963,7 @@ const unsigned char _Py_M__importlib_external[] = { 167,0,0,0,218,11,99,108,97,115,115,109,101,116,104,111, 100,114,166,0,0,0,114,172,0,0,0,114,175,0,0,0, 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,162,0,0,0,57,2, + 4,0,0,0,114,5,0,0,0,114,162,0,0,0,58,2, 0,0,115,20,0,0,0,12,2,6,3,6,3,6,2,6, 2,18,7,18,15,3,1,21,15,3,1,114,162,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0, @@ -1001,7 +1001,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,119,0,0,0,114,94,0,0,0,90,13, 102,105,108,101,110,97,109,101,95,98,97,115,101,90,9,116, 97,105,108,95,110,97,109,101,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,153,0,0,0,126,2,0,0, + 0,0,114,5,0,0,0,114,153,0,0,0,127,2,0,0, 115,8,0,0,0,0,3,25,1,22,1,19,1,122,24,95, 76,111,97,100,101,114,66,97,115,105,99,115,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1012,7 +1012,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,110,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, 5,0,0,0,218,13,99,114,101,97,116,101,95,109,111,100, - 117,108,101,134,2,0,0,115,0,0,0,0,122,27,95,76, + 117,108,101,135,2,0,0,115,0,0,0,0,122,27,95,76, 111,97,100,101,114,66,97,115,105,99,115,46,99,114,101,97, 116,101,95,109,111,100,117,108,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,4,0,0,0,67,0,0,0,115,80, @@ -1033,7 +1033,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,99,114,111,0,0,0,41,3,114,100,0,0,0,218,6, 109,111,100,117,108,101,114,140,0,0,0,114,4,0,0,0, 114,4,0,0,0,114,5,0,0,0,218,11,101,120,101,99, - 95,109,111,100,117,108,101,137,2,0,0,115,10,0,0,0, + 95,109,111,100,117,108,101,138,2,0,0,115,10,0,0,0, 0,2,18,1,12,1,9,1,15,1,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1044,14 +1044,14 @@ const unsigned char _Py_M__importlib_external[] = { 114,0,0,0,218,17,95,108,111,97,100,95,109,111,100,117, 108,101,95,115,104,105,109,41,2,114,100,0,0,0,114,119, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,145, + 0,0,218,11,108,111,97,100,95,109,111,100,117,108,101,146, 2,0,0,115,2,0,0,0,0,2,122,25,95,76,111,97, 100,101,114,66,97,115,105,99,115,46,108,111,97,100,95,109, 111,100,117,108,101,78,41,8,114,105,0,0,0,114,104,0, 0,0,114,106,0,0,0,114,107,0,0,0,114,153,0,0, 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,178,0,0,0,121,2,0,0,115,10,0, + 5,0,0,0,114,178,0,0,0,122,2,0,0,115,10,0, 0,0,12,3,6,2,12,8,12,3,12,8,114,178,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,4,0, 0,0,64,0,0,0,115,106,0,0,0,101,0,0,90,1, @@ -1079,7 +1079,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,1,218,7,73,79,69,114,114,111,114,41,2,114,100,0, 0,0,114,35,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,10,112,97,116,104,95,109,116,105, - 109,101,152,2,0,0,115,2,0,0,0,0,6,122,23,83, + 109,101,153,2,0,0,115,2,0,0,0,0,6,122,23,83, 111,117,114,99,101,76,111,97,100,101,114,46,112,97,116,104, 95,109,116,105,109,101,99,2,0,0,0,0,0,0,0,2, 0,0,0,3,0,0,0,67,0,0,0,115,19,0,0,0, @@ -1114,7 +1114,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,114,126,0,0,0,41,1,114,190,0,0,0, 41,2,114,100,0,0,0,114,35,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,10,112,97,116, - 104,95,115,116,97,116,115,160,2,0,0,115,2,0,0,0, + 104,95,115,116,97,116,115,161,2,0,0,115,2,0,0,0, 0,11,122,23,83,111,117,114,99,101,76,111,97,100,101,114, 46,112,97,116,104,95,115,116,97,116,115,99,4,0,0,0, 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, @@ -1138,7 +1138,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,90,0,0,0,90,10,99,97,99,104,101,95,112,97,116, 104,114,53,0,0,0,114,4,0,0,0,114,4,0,0,0, 114,5,0,0,0,218,15,95,99,97,99,104,101,95,98,121, - 116,101,99,111,100,101,173,2,0,0,115,2,0,0,0,0, + 116,101,99,111,100,101,174,2,0,0,115,2,0,0,0,0, 8,122,28,83,111,117,114,99,101,76,111,97,100,101,114,46, 95,99,97,99,104,101,95,98,121,116,101,99,111,100,101,99, 3,0,0,0,0,0,0,0,3,0,0,0,1,0,0,0, @@ -1155,7 +1155,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,32,78,114,4,0,0,0,41,3,114, 100,0,0,0,114,35,0,0,0,114,53,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,192,0, - 0,0,183,2,0,0,115,0,0,0,0,122,21,83,111,117, + 0,0,184,2,0,0,115,0,0,0,0,122,21,83,111,117, 114,99,101,76,111,97,100,101,114,46,115,101,116,95,100,97, 116,97,99,2,0,0,0,0,0,0,0,5,0,0,0,16, 0,0,0,67,0,0,0,115,105,0,0,0,124,0,0,106, @@ -1177,7 +1177,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,119,0,0,0,114,35,0,0,0,114,147,0, 0,0,218,3,101,120,99,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,10,103,101,116,95,115,111,117,114, - 99,101,190,2,0,0,115,14,0,0,0,0,2,15,1,3, + 99,101,191,2,0,0,115,14,0,0,0,0,2,15,1,3, 1,19,1,18,1,9,1,31,1,122,23,83,111,117,114,99, 101,76,111,97,100,101,114,46,103,101,116,95,115,111,117,114, 99,101,218,9,95,111,112,116,105,109,105,122,101,114,29,0, @@ -1199,7 +1199,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,101,41,4,114,100,0,0,0,114,53,0,0,0,114,35, 0,0,0,114,197,0,0,0,114,4,0,0,0,114,4,0, 0,0,114,5,0,0,0,218,14,115,111,117,114,99,101,95, - 116,111,95,99,111,100,101,200,2,0,0,115,4,0,0,0, + 116,111,95,99,111,100,101,201,2,0,0,115,4,0,0,0, 0,5,21,1,122,27,83,111,117,114,99,101,76,111,97,100, 101,114,46,115,111,117,114,99,101,95,116,111,95,99,111,100, 101,99,2,0,0,0,0,0,0,0,10,0,0,0,43,0, @@ -1260,7 +1260,7 @@ const unsigned char _Py_M__importlib_external[] = { 89,0,0,0,218,2,115,116,114,53,0,0,0,218,10,98, 121,116,101,115,95,100,97,116,97,114,147,0,0,0,90,11, 99,111,100,101,95,111,98,106,101,99,116,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,208, + 114,4,0,0,0,114,5,0,0,0,114,181,0,0,0,209, 2,0,0,115,78,0,0,0,0,7,15,1,6,1,3,1, 16,1,13,1,11,2,3,1,19,1,13,1,5,2,16,1, 3,1,19,1,13,1,5,2,3,1,9,1,12,1,13,1, @@ -1273,7 +1273,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,192,0,0,0,114,196,0,0,0,114,200,0, 0,0,114,181,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,188,0,0,0, - 150,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, + 151,2,0,0,115,14,0,0,0,12,2,12,8,12,13,12, 10,12,7,12,10,18,8,114,188,0,0,0,99,0,0,0, 0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0, 0,115,112,0,0,0,101,0,0,90,1,0,100,0,0,90, @@ -1301,7 +1301,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,102,105,110,100,101,114,46,78,41,2,114, 98,0,0,0,114,35,0,0,0,41,3,114,100,0,0,0, 114,119,0,0,0,114,35,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,179,0,0,0,9,3, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,10,3, 0,0,115,4,0,0,0,0,3,9,1,122,19,70,105,108, 101,76,111,97,100,101,114,46,95,95,105,110,105,116,95,95, 99,2,0,0,0,0,0,0,0,2,0,0,0,2,0,0, @@ -1311,7 +1311,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,218,9,95,95,99,108,97,115,115,95,95,114,111,0,0, 0,41,2,114,100,0,0,0,218,5,111,116,104,101,114,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,95,101,113,95,95,15,3,0,0,115,4,0,0,0,0, + 95,95,101,113,95,95,16,3,0,0,115,4,0,0,0,0, 1,18,1,122,17,70,105,108,101,76,111,97,100,101,114,46, 95,95,101,113,95,95,99,1,0,0,0,0,0,0,0,1, 0,0,0,3,0,0,0,67,0,0,0,115,26,0,0,0, @@ -1319,7 +1319,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,106,2,0,131,1,0,65,83,41,1,78,41,3,218, 4,104,97,115,104,114,98,0,0,0,114,35,0,0,0,41, 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,19, + 114,5,0,0,0,218,8,95,95,104,97,115,104,95,95,20, 3,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, 76,111,97,100,101,114,46,95,95,104,97,115,104,95,95,99, 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, @@ -1334,7 +1334,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,32,41,3,218,5,115,117,112,101,114, 114,204,0,0,0,114,187,0,0,0,41,2,114,100,0,0, 0,114,119,0,0,0,41,1,114,205,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,187,0,0,0,22,3,0,0, + 0,0,114,5,0,0,0,114,187,0,0,0,23,3,0,0, 115,2,0,0,0,0,10,122,22,70,105,108,101,76,111,97, 100,101,114,46,108,111,97,100,95,109,111,100,117,108,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, @@ -1345,7 +1345,7 @@ const unsigned char _Py_M__importlib_external[] = { 98,121,32,116,104,101,32,102,105,110,100,101,114,46,41,1, 114,35,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,151,0,0,0,34,3,0,0,115,2,0,0,0,0,3, + 114,151,0,0,0,35,3,0,0,115,2,0,0,0,0,3, 122,23,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,102,105,108,101,110,97,109,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,9,0,0,0,67,0,0,0,115,42, @@ -1358,14 +1358,14 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,50,0,0,0,90,4,114,101,97,100,41,3, 114,100,0,0,0,114,35,0,0,0,114,54,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,194, - 0,0,0,39,3,0,0,115,4,0,0,0,0,2,21,1, + 0,0,0,40,3,0,0,115,4,0,0,0,0,2,21,1, 122,19,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,100,97,116,97,41,11,114,105,0,0,0,114,104,0,0, 0,114,106,0,0,0,114,107,0,0,0,114,179,0,0,0, 114,207,0,0,0,114,209,0,0,0,114,116,0,0,0,114, 187,0,0,0,114,151,0,0,0,114,194,0,0,0,114,4, 0,0,0,114,4,0,0,0,41,1,114,205,0,0,0,114, - 5,0,0,0,114,204,0,0,0,4,3,0,0,115,14,0, + 5,0,0,0,114,204,0,0,0,5,3,0,0,115,14,0, 0,0,12,3,6,2,12,6,12,4,12,3,24,12,18,5, 114,204,0,0,0,99,0,0,0,0,0,0,0,0,0,0, 0,0,4,0,0,0,64,0,0,0,115,64,0,0,0,101, @@ -1388,7 +1388,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,39,0,0,0,218,8,115,116,95,109,116,105,109,101,90, 7,115,116,95,115,105,122,101,41,3,114,100,0,0,0,114, 35,0,0,0,114,202,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,191,0,0,0,49,3,0, + 0,0,0,114,5,0,0,0,114,191,0,0,0,50,3,0, 0,115,4,0,0,0,0,2,12,1,122,27,83,111,117,114, 99,101,70,105,108,101,76,111,97,100,101,114,46,112,97,116, 104,95,115,116,97,116,115,99,4,0,0,0,0,0,0,0, @@ -1399,7 +1399,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,0,0,0,114,192,0,0,0,41,5,114,100,0,0,0, 114,90,0,0,0,114,89,0,0,0,114,53,0,0,0,114, 42,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,193,0,0,0,54,3,0,0,115,4,0,0, + 0,0,0,114,193,0,0,0,55,3,0,0,115,4,0,0, 0,0,2,12,1,122,32,83,111,117,114,99,101,70,105,108, 101,76,111,97,100,101,114,46,95,99,97,99,104,101,95,98, 121,116,101,99,111,100,101,114,214,0,0,0,105,182,1,0, @@ -1438,7 +1438,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,214,0,0,0,218,6,112,97,114,101,110,116,114,94, 0,0,0,114,27,0,0,0,114,23,0,0,0,114,195,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,192,0,0,0,59,3,0,0,115,42,0,0,0,0, + 0,114,192,0,0,0,60,3,0,0,115,42,0,0,0,0, 2,18,1,6,2,22,1,18,1,17,2,19,1,15,1,3, 1,17,1,13,2,7,1,18,3,9,1,10,1,27,1,3, 1,16,1,20,1,18,2,12,1,122,25,83,111,117,114,99, @@ -1447,7 +1447,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,106,0,0,0,114,107,0,0,0,114,191,0,0,0, 114,193,0,0,0,114,192,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,212, - 0,0,0,45,3,0,0,115,8,0,0,0,12,2,6,2, + 0,0,0,46,3,0,0,115,8,0,0,0,12,2,6,2, 12,5,12,5,114,212,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,46, 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, @@ -1469,7 +1469,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,141,0,0,0,41,5,114,100,0,0,0,114,119,0, 0,0,114,35,0,0,0,114,53,0,0,0,114,203,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,94,3,0,0,115,8,0,0,0,0,1, + 114,181,0,0,0,95,3,0,0,115,8,0,0,0,0,1, 15,1,15,1,24,1,122,29,83,111,117,114,99,101,108,101, 115,115,70,105,108,101,76,111,97,100,101,114,46,103,101,116, 95,99,111,100,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1479,13 +1479,13 @@ const unsigned char _Py_M__importlib_external[] = { 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, - 196,0,0,0,100,3,0,0,115,2,0,0,0,0,2,122, + 196,0,0,0,101,3,0,0,115,2,0,0,0,0,2,122, 31,83,111,117,114,99,101,108,101,115,115,70,105,108,101,76, 111,97,100,101,114,46,103,101,116,95,115,111,117,114,99,101, 78,41,6,114,105,0,0,0,114,104,0,0,0,114,106,0, 0,0,114,107,0,0,0,114,181,0,0,0,114,196,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,217,0,0,0,90,3,0,0,115,6, + 114,5,0,0,0,114,217,0,0,0,91,3,0,0,115,6, 0,0,0,12,2,6,2,12,6,114,217,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, 0,0,0,115,136,0,0,0,101,0,0,90,1,0,100,0, @@ -1510,7 +1510,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,0,100,0,0,83,41,1,78,41,2,114,98,0,0,0, 114,35,0,0,0,41,3,114,100,0,0,0,114,98,0,0, 0,114,35,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,5,0,0,0,114,179,0,0,0,117,3,0,0,115,4, + 114,5,0,0,0,114,179,0,0,0,118,3,0,0,115,4, 0,0,0,0,1,9,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,105, 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, @@ -1520,7 +1520,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,1,78,41,2,114,205,0,0,0,114,111,0,0,0, 41,2,114,100,0,0,0,114,206,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,207,0,0,0, - 121,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, + 122,3,0,0,115,4,0,0,0,0,1,18,1,122,26,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,95,95,101,113,95,95,99,1,0,0,0,0,0, 0,0,1,0,0,0,3,0,0,0,67,0,0,0,115,26, @@ -1528,7 +1528,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,124,0,0,106,2,0,131,1,0,65,83,41,1,78, 41,3,114,208,0,0,0,114,98,0,0,0,114,35,0,0, 0,41,1,114,100,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,209,0,0,0,125,3,0,0, + 0,0,114,5,0,0,0,114,209,0,0,0,126,3,0,0, 115,2,0,0,0,0,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,104, 97,115,104,95,95,99,2,0,0,0,0,0,0,0,3,0, @@ -1546,7 +1546,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, 0,0,0,41,3,114,100,0,0,0,114,158,0,0,0,114, 184,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,180,0,0,0,128,3,0,0,115,10,0,0, + 0,0,0,114,180,0,0,0,129,3,0,0,115,10,0,0, 0,0,2,6,1,15,1,9,1,16,1,122,33,69,120,116, 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, 46,99,114,101,97,116,101,95,109,111,100,117,108,101,99,2, @@ -1564,7 +1564,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,109,105,99,114,129,0,0,0,114,98,0,0,0,114,35, 0,0,0,41,2,114,100,0,0,0,114,184,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,185, - 0,0,0,136,3,0,0,115,6,0,0,0,0,2,19,1, + 0,0,0,137,3,0,0,115,6,0,0,0,0,2,19,1, 9,1,122,31,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, @@ -1582,7 +1582,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,2,114,179,0,0,0,78,114,4,0,0,0,41,2,114, 22,0,0,0,218,6,115,117,102,102,105,120,41,1,218,9, 102,105,108,101,95,110,97,109,101,114,4,0,0,0,114,5, - 0,0,0,250,9,60,103,101,110,101,120,112,114,62,145,3, + 0,0,0,250,9,60,103,101,110,101,120,112,114,62,146,3, 0,0,115,2,0,0,0,6,1,122,49,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,105, 115,95,112,97,99,107,97,103,101,46,60,108,111,99,97,108, @@ -1591,7 +1591,7 @@ const unsigned char _Py_M__importlib_external[] = { 88,84,69,78,83,73,79,78,95,83,85,70,70,73,88,69, 83,41,2,114,100,0,0,0,114,119,0,0,0,114,4,0, 0,0,41,1,114,220,0,0,0,114,5,0,0,0,114,153, - 0,0,0,142,3,0,0,115,6,0,0,0,0,2,19,1, + 0,0,0,143,3,0,0,115,6,0,0,0,0,2,19,1, 18,1,122,30,69,120,116,101,110,115,105,111,110,70,105,108, 101,76,111,97,100,101,114,46,105,115,95,112,97,99,107,97, 103,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1602,7 +1602,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,97,116,101,32,97,32,99,111,100,101,32,111,98,106,101, 99,116,46,78,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,181,0,0,0,148,3,0,0,115,2,0, + 5,0,0,0,114,181,0,0,0,149,3,0,0,115,2,0, 0,0,0,2,122,28,69,120,116,101,110,115,105,111,110,70, 105,108,101,76,111,97,100,101,114,46,103,101,116,95,99,111, 100,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1612,7 +1612,7 @@ const unsigned char _Py_M__importlib_external[] = { 117,108,101,115,32,104,97,118,101,32,110,111,32,115,111,117, 114,99,101,32,99,111,100,101,46,78,114,4,0,0,0,41, 2,114,100,0,0,0,114,119,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,152, + 114,4,0,0,0,114,5,0,0,0,114,196,0,0,0,153, 3,0,0,115,2,0,0,0,0,2,122,30,69,120,116,101, 110,115,105,111,110,70,105,108,101,76,111,97,100,101,114,46, 103,101,116,95,115,111,117,114,99,101,99,2,0,0,0,0, @@ -1624,7 +1624,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,32,102,105,110,100,101,114,46,41,1,114,35,0,0,0, 41,2,114,100,0,0,0,114,119,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,151,0,0,0, - 156,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, + 157,3,0,0,115,2,0,0,0,0,3,122,32,69,120,116, 101,110,115,105,111,110,70,105,108,101,76,111,97,100,101,114, 46,103,101,116,95,102,105,108,101,110,97,109,101,78,41,14, 114,105,0,0,0,114,104,0,0,0,114,106,0,0,0,114, @@ -1632,7 +1632,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,180,0,0,0,114,185,0,0,0,114,153,0, 0,0,114,181,0,0,0,114,196,0,0,0,114,116,0,0, 0,114,151,0,0,0,114,4,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,109, + 114,4,0,0,0,114,5,0,0,0,114,218,0,0,0,110, 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, @@ -1677,7 +1677,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,169,3,0,0,115,8,0,0,0,0,1, + 114,179,0,0,0,170,3,0,0,115,8,0,0,0,0,1, 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, @@ -1696,7 +1696,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, - 97,109,101,115,175,3,0,0,115,8,0,0,0,0,2,27, + 97,109,101,115,176,3,0,0,115,8,0,0,0,0,2,27, 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, @@ -1709,7 +1709,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,227,0,0,0,185,3,0,0,115,4,0, + 5,0,0,0,114,227,0,0,0,186,3,0,0,115,4,0, 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, @@ -1727,7 +1727,7 @@ const unsigned char _Py_M__importlib_external[] = { 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, - 101,99,97,108,99,117,108,97,116,101,189,3,0,0,115,16, + 101,99,97,108,99,117,108,97,116,101,190,3,0,0,115,16, 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, @@ -1736,7 +1736,7 @@ const unsigned char _Py_M__importlib_external[] = { 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,8,95,95,105,116,101,114,95,95,202,3,0,0,115,2, + 218,8,95,95,105,116,101,114,95,95,203,3,0,0,115,2, 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, @@ -1744,7 +1744,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, - 95,108,101,110,95,95,205,3,0,0,115,2,0,0,0,0, + 95,108,101,110,95,95,206,3,0,0,115,2,0,0,0,0, 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, @@ -1753,7 +1753,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,95,114,101,112,114,95,95,208,3,0,0,115,2,0,0, + 95,95,114,101,112,114,95,95,209,3,0,0,115,2,0,0, 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, @@ -1761,7 +1761,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, - 99,111,110,116,97,105,110,115,95,95,211,3,0,0,115,2, + 99,111,110,116,97,105,110,115,95,95,212,3,0,0,115,2, 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, @@ -1769,7 +1769,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,214, + 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,215, 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, @@ -1777,7 +1777,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,224,0,0,0,162,3, + 4,0,0,0,114,5,0,0,0,114,224,0,0,0,163,3, 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, @@ -1797,7 +1797,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, - 220,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, + 221,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, @@ -1814,21 +1814,21 @@ const unsigned char _Py_M__importlib_external[] = { 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, - 101,95,114,101,112,114,223,3,0,0,115,2,0,0,0,0, + 101,95,114,101,112,114,224,3,0,0,115,2,0,0,0,0, 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,153,0,0,0,232,3,0,0,115,2,0,0,0,0, + 0,114,153,0,0,0,233,3,0,0,115,2,0,0,0,0, 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,235,3,0,0,115,2,0, + 5,0,0,0,114,196,0,0,0,236,3,0,0,115,2,0, 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, @@ -1838,7 +1838,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,238,3,0,0,115,2,0,0,0,0,1, + 114,181,0,0,0,239,3,0,0,115,2,0,0,0,0,1, 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, @@ -1847,14 +1847,14 @@ const unsigned char _Py_M__importlib_external[] = { 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,241,3,0, + 0,0,0,114,5,0,0,0,114,180,0,0,0,242,3,0, 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,185,0,0,0,244,3,0,0, + 0,0,114,5,0,0,0,114,185,0,0,0,245,3,0,0, 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, @@ -1873,7 +1873,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,187,0,0,0,247,3,0,0,115,6,0,0,0,0, + 0,114,187,0,0,0,248,3,0,0,115,6,0,0,0,0, 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, @@ -1881,7 +1881,7 @@ const unsigned char _Py_M__importlib_external[] = { 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,241,0,0,0,219,3,0,0,115, + 0,114,5,0,0,0,114,241,0,0,0,220,3,0,0,115, 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, @@ -1919,7 +1919,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,9,4,0,0,115,6,0,0, + 0,0,0,114,244,0,0,0,10,4,0,0,115,6,0,0, 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, @@ -1944,7 +1944,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,95,112,97,116,104,95,104,111,111,107,115,17,4, + 0,218,11,95,112,97,116,104,95,104,111,111,107,115,18,4, 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, @@ -1977,7 +1977,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,34,4,0,0,115,22,0, + 116,101,114,95,99,97,99,104,101,35,4,0,0,115,22,0, 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, @@ -1997,7 +1997,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, - 103,101,116,95,115,112,101,99,56,4,0,0,115,18,0,0, + 103,101,116,95,115,112,101,99,57,4,0,0,115,18,0,0, 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, @@ -2033,7 +2033,7 @@ const unsigned char _Py_M__importlib_external[] = { 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, - 101,116,95,115,112,101,99,71,4,0,0,115,40,0,0,0, + 101,116,95,115,112,101,99,72,4,0,0,115,40,0,0,0, 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, @@ -2060,7 +2060,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,103, + 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,104, 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, @@ -2083,7 +2083,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, - 125,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, + 126,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, @@ -2091,7 +2091,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, - 5,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, + 6,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, @@ -2140,7 +2140,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 154,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, + 155,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, @@ -2152,7 +2152,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, - 0,0,0,114,5,0,0,0,114,179,0,0,0,148,4,0, + 0,0,0,114,5,0,0,0,114,179,0,0,0,149,4,0, 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, @@ -2163,7 +2163,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, - 0,0,162,4,0,0,115,2,0,0,0,0,2,122,28,70, + 0,0,163,4,0,0,115,2,0,0,0,0,2,122,28,70, 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, @@ -2187,7 +2187,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, - 168,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, + 169,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, @@ -2198,7 +2198,7 @@ const unsigned char _Py_M__importlib_external[] = { 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,255,0,0,0,180,4,0,0,115,6,0, + 5,0,0,0,114,255,0,0,0,181,4,0,0,115,6,0, 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, @@ -2262,7 +2262,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, - 185,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, + 186,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, @@ -2298,7 +2298,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, - 9,60,115,101,116,99,111,109,112,62,4,5,0,0,115,2, + 9,60,115,101,116,99,111,109,112,62,5,5,0,0,115,2, 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, @@ -2315,7 +2315,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,7,1,0,0,231,4,0,0,115, + 0,114,5,0,0,0,114,7,1,0,0,232,4,0,0,115, 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, @@ -2354,7 +2354,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, - 70,105,108,101,70,105,110,100,101,114,16,5,0,0,115,6, + 70,105,108,101,70,105,110,100,101,114,17,5,0,0,115,6, 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, @@ -2362,7 +2362,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, - 97,116,104,95,104,111,111,107,6,5,0,0,115,4,0,0, + 97,116,104,95,104,111,111,107,7,5,0,0,115,4,0,0, 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, @@ -2371,7 +2371,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, - 0,24,5,0,0,115,2,0,0,0,0,1,122,19,70,105, + 0,25,5,0,0,115,2,0,0,0,0,1,122,19,70,105, 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, @@ -2379,7 +2379,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,0,1,0,0,139,4,0,0,115,20,0,0,0, + 0,0,114,0,1,0,0,140,4,0,0,115,20,0,0,0, 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, @@ -2405,7 +2405,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, - 117,112,95,109,111,100,117,108,101,30,5,0,0,115,34,0, + 117,112,95,109,111,100,117,108,101,31,5,0,0,115,34,0, 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, @@ -2426,7 +2426,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,155,0,0,0,53,5,0,0,115,8,0,0,0,0,5, + 114,155,0,0,0,54,5,0,0,115,8,0,0,0,0,5, 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, @@ -2488,7 +2488,7 @@ const unsigned char _Py_M__importlib_external[] = { 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 89,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, + 90,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, @@ -2518,7 +2518,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,115,101,116,117,112,64,5,0,0,115,82,0,0,0,0, + 95,115,101,116,117,112,65,5,0,0,115,82,0,0,0,0, 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, @@ -2544,7 +2544,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,105,110,115,116,97,108,108,132,5,0,0,115,16,0,0, + 95,105,110,115,116,97,108,108,133,5,0,0,115,16,0,0, 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, @@ -2571,11 +2571,12 @@ const unsigned char _Py_M__importlib_external[] = { 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, - 101,62,8,0,0,0,115,98,0,0,0,6,17,6,3,12, + 101,62,8,0,0,0,115,102,0,0,0,6,17,6,3,12, 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, - 7,15,22,15,111,22,1,18,2,6,1,6,2,9,2,9, + 7,15,22,15,112,22,1,18,2,6,1,6,2,9,2,9, 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, - 57,19,42,19,134,19,147,15,23,12,11,12,68, + 57,19,42,19,127,0,7,19,127,0,20,15,23,12,11,12, + 68, }; diff --git a/Python/peephole.c b/Python/peephole.c index 4bf786e..c33bf1a 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -346,19 +346,19 @@ markblocks(unsigned char *code, Py_ssize_t len) single basic block. All transformations keep the code size the same or smaller. For those that reduce size, the gaps are initially filled with NOPs. Later those NOPs are removed and the jump addresses retargeted in - a single pass. Line numbering is adjusted accordingly. */ + a single pass. Code offset is adjusted accordingly. */ PyObject * PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, - PyObject *lineno_obj) + PyObject *lnotab_obj) { Py_ssize_t i, j, codelen; int nops, h, adj; int tgt, tgttgt, opcode; unsigned char *codestr = NULL; - unsigned char *lineno; + unsigned char *lnotab; int *addrmap = NULL; - int new_line, cum_orig_line, last_line; + int cum_orig_offset, last_offset; Py_ssize_t tabsiz; PyObject **const_stack = NULL; Py_ssize_t *load_const_stack = NULL; @@ -371,12 +371,17 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, if (PyErr_Occurred()) goto exitError; - /* Bypass optimization when the lineno table is too complex */ - assert(PyBytes_Check(lineno_obj)); - lineno = (unsigned char*)PyBytes_AS_STRING(lineno_obj); - tabsiz = PyBytes_GET_SIZE(lineno_obj); - if (memchr(lineno, 255, tabsiz) != NULL) + /* Bypass optimization when the lnotab table is too complex */ + assert(PyBytes_Check(lnotab_obj)); + lnotab = (unsigned char*)PyBytes_AS_STRING(lnotab_obj); + tabsiz = PyBytes_GET_SIZE(lnotab_obj); + assert(tabsiz == 0 || Py_REFCNT(lnotab_obj) == 1); + if (memchr(lnotab, 255, tabsiz) != NULL) { + /* 255 value are used for multibyte bytecode instructions */ goto exitUnchanged; + } + /* Note: -128 and 127 special values for line number delta are ok, + the peephole optimizer doesn't modify line numbers. */ /* Avoid situations where jump retargeting could overflow */ assert(PyBytes_Check(code)); @@ -663,21 +668,24 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, } } - /* Fixup linenotab */ + /* Fixup lnotab */ for (i=0, nops=0 ; i new code offset */ addrmap[i] = (int)(i - nops); if (codestr[i] == NOP) nops++; } - cum_orig_line = 0; - last_line = 0; + cum_orig_offset = 0; + last_offset = 0; for (i=0 ; i < tabsiz ; i+=2) { - cum_orig_line += lineno[i]; - new_line = addrmap[cum_orig_line]; - assert (new_line - last_line < 255); - lineno[i] =((unsigned char)(new_line - last_line)); - last_line = new_line; + int offset_delta, new_offset; + cum_orig_offset += lnotab[i]; + new_offset = addrmap[cum_orig_offset]; + offset_delta = new_offset - last_offset; + assert(0 <= offset_delta && offset_delta <= 255); + lnotab[i] = (unsigned char)offset_delta; + last_offset = new_offset; } /* Remove NOPs and fixup jump targets */ @@ -727,12 +735,9 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, exitUnchanged: CONST_STACK_DELETE(); - if (blocks != NULL) - PyMem_Free(blocks); - if (addrmap != NULL) - PyMem_Free(addrmap); - if (codestr != NULL) - PyMem_Free(codestr); + PyMem_Free(blocks); + PyMem_Free(addrmap); + PyMem_Free(codestr); Py_XINCREF(code); return code; } -- cgit v0.12 From 9f7893955298a5691ee9d9c65d98bfc7d1eb34f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 21 Jan 2016 18:12:29 +0100 Subject: Issue #26107: Fix typo in Objects/lnotab_notes.txt Double parenthesis --- Objects/lnotab_notes.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/lnotab_notes.txt b/Objects/lnotab_notes.txt index 8af10db..5153757 100644 --- a/Objects/lnotab_notes.txt +++ b/Objects/lnotab_notes.txt @@ -22,7 +22,7 @@ look like: 0, 1, 6, 1, 44, 5, 300, 200, 11, 1 The above doesn't really work, but it's a start. An unsigned byte (byte code -offset)) can't hold negative values, or values larger than 255, a signed byte +offset) can't hold negative values, or values larger than 255, a signed byte (line number) can't hold values larger than 127 or less than -128, and the above example contains two such values. So we make two tweaks: @@ -95,16 +95,16 @@ which compiles to this: 6 POP_JUMP_IF_FALSE 17 3 9 LOAD_CONST 1 (1) - 12 PRINT_ITEM + 12 PRINT_ITEM - 4 13 BREAK_LOOP + 4 13 BREAK_LOOP 14 JUMP_ABSOLUTE 3 - >> 17 POP_BLOCK + >> 17 POP_BLOCK 6 18 LOAD_CONST 2 (2) - 21 PRINT_ITEM + 21 PRINT_ITEM >> 22 LOAD_CONST 0 (None) - 25 RETURN_VALUE + 25 RETURN_VALUE If 'a' is false, execution will jump to the POP_BLOCK instruction at offset 17 and the co_lnotab will claim that execution has moved to line 4, which is wrong. -- cgit v0.12 From e3560a7dc9eeac324ff407588cb3f0b36ffe5c6e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:22:07 +0100 Subject: site: error on sitecustomize import error Issue #26099: The site module now writes an error into stderr if sitecustomize module can be imported but executing the module raise an ImportError. Same change for usercustomize. --- Lib/site.py | 20 ++++++++++++++------ Misc/NEWS | 4 ++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 3a8d1c3..13ec8fa 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -504,9 +504,13 @@ def venv(known_paths): def execsitecustomize(): """Run custom site specific code, if available.""" try: - import sitecustomize - except ImportError: - pass + try: + import sitecustomize + except ImportError as exc: + if exc.name == 'sitecustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) @@ -520,9 +524,13 @@ def execsitecustomize(): def execusercustomize(): """Run custom user specific code, if available.""" try: - import usercustomize - except ImportError: - pass + try: + import usercustomize + except ImportError as exc: + if exc.name == 'usercustomize': + pass + else: + raise except Exception as err: if os.environ.get("PYTHONVERBOSE"): sys.excepthook(*sys.exc_info()) diff --git a/Misc/NEWS b/Misc/NEWS index 01e933c..4e3cc96 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -146,6 +146,10 @@ Core and Builtins Library ------- +- Issue #26099: The site module now writes an error into stderr if + sitecustomize module can be imported but executing the module raise an + ImportError. Same change for usercustomize. + - Issue #26147: xmlrpc now works with strings not encodable with used non-UTF-8 encoding. -- cgit v0.12 From efb2413ce82acaa5dec43a8cb14aa7cdf2352fb1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 12:33:12 +0100 Subject: code_richcompare() now uses the constants types Issue #25843: When compiling code, don't merge constants if they are equal but have a different types. For example, "f1, f2 = lambda: 1, lambda: 1.0" is now correctly compiled to two different functions: f1() returns 1 (int) and f2() returns 1.0 (int), even if 1 and 1.0 are equal. Add a new _PyCode_ConstantKey() private function. --- Include/code.h | 11 +++- Lib/test/test_compile.py | 82 ++++++++++++++++++++++++++++ Misc/NEWS | 6 ++ Objects/codeobject.c | 139 ++++++++++++++++++++++++++++++++++++++++++++++- Python/compile.c | 58 ++++---------------- 5 files changed, 246 insertions(+), 50 deletions(-) diff --git a/Include/code.h b/Include/code.h index 3ce2084..a300ead 100644 --- a/Include/code.h +++ b/Include/code.h @@ -108,12 +108,21 @@ typedef struct _addr_pair { int ap_upper; } PyAddrPair; +#ifndef Py_LIMITED_API /* Update *bounds to describe the first and one-past-the-last instructions in the same line as lasti. Return the number of that line. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds); + +/* Create a comparable key used to compare constants taking in account the + * object type. It is used to make sure types are not coerced (e.g., float and + * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms + * + * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items) + * depending on the type and the value. The type is the first item to not + * compare bytes and str which can raise a BytesWarning exception. */ +PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj); #endif PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 8935c65..e0fdee3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -572,6 +572,88 @@ if 1: exec(memoryview(b"ax = 123")[1:-1], namespace) self.assertEqual(namespace['x'], 12) + def check_constant(self, func, expected): + for const in func.__code__.co_consts: + if repr(const) == repr(expected): + break + else: + self.fail("unable to find constant %r in %r" + % (expected, func.__code__.co_consts)) + + # Merging equal constants is not a strict requirement for the Python + # semantics, it's a more an implementation detail. + @support.cpython_only + def test_merge_constants(self): + # Issue #25843: compile() must merge constants which are equal + # and have the same type. + + def check_same_constant(const): + ns = {} + code = "f1, f2 = lambda: %r, lambda: %r" % (const, const) + exec(code, ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, const) + self.assertEqual(repr(f1()), repr(const)) + + check_same_constant(None) + check_same_constant(0) + check_same_constant(0.0) + check_same_constant(b'abc') + check_same_constant('abc') + + # Note: "lambda: ..." emits "LOAD_CONST Ellipsis", + # whereas "lambda: Ellipsis" emits "LOAD_GLOBAL Ellipsis" + f1, f2 = lambda: ..., lambda: ... + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, Ellipsis) + self.assertEqual(repr(f1()), repr(Ellipsis)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0} + self.assertIs(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.assertTrue(f1(0)) + + def test_dont_merge_constants(self): + # Issue #25843: compile() must not merge constants which are equal + # but have a different type. + + def check_different_constants(const1, const2): + ns = {} + exec("f1, f2 = lambda: %r, lambda: %r" % (const1, const2), ns) + f1 = ns['f1'] + f2 = ns['f2'] + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, const1) + self.check_constant(f2, const2) + self.assertEqual(repr(f1()), repr(const1)) + self.assertEqual(repr(f2()), repr(const2)) + + check_different_constants(0, 0.0) + check_different_constants(+0.0, -0.0) + check_different_constants((0,), (0.0,)) + + # check_different_constants() cannot be used because repr(-0j) is + # '(-0-0j)', but when '(-0-0j)' is evaluated to 0j: we loose the sign. + f1, f2 = lambda: +0.0j, lambda: -0.0j + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, +0.0j) + self.check_constant(f2, -0.0j) + self.assertEqual(repr(f1()), repr(+0.0j)) + self.assertEqual(repr(f2()), repr(-0.0j)) + + # {0} is converted to a constant frozenset({0}) by the peephole + # optimizer + f1, f2 = lambda x: x in {0}, lambda x: x in {0.0} + self.assertIsNot(f1.__code__, f2.__code__) + self.check_constant(f1, frozenset({0})) + self.check_constant(f2, frozenset({0.0})) + self.assertTrue(f1(0)) + self.assertTrue(f2(0.0)) + class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object diff --git a/Misc/NEWS b/Misc/NEWS index 4e3cc96..409feae 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,12 @@ Release date: tba Core and Builtins ----------------- +- Issue #25843: When compiling code, don't merge constants if they are equal + but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` + is now correctly compiled to two different functions: ``f1()`` returns ``1`` + (``int``) and ``f2()`` returns ``1.0`` (``int``), even if ``1`` and ``1.0`` + are equal. + - Issue #26107: The format of the ``co_lnotab`` attribute of code objects changes to support negative line number delta. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0f03dfe..e3d28e3 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -409,11 +409,135 @@ code_repr(PyCodeObject *co) } } +PyObject* +_PyCode_ConstantKey(PyObject *op) +{ + PyObject *key; + + /* Py_None and Py_Ellipsis are singleton */ + if (op == Py_None || op == Py_Ellipsis + || PyLong_CheckExact(op) + || PyBool_Check(op) + || PyBytes_CheckExact(op) + || PyUnicode_CheckExact(op) + /* code_richcompare() uses _PyCode_ConstantKey() internally */ + || PyCode_Check(op)) { + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + else if (PyFloat_CheckExact(op)) { + double d = PyFloat_AS_DOUBLE(op); + /* all we need is to make the tuple different in either the 0.0 + * or -0.0 case from all others, just to avoid the "coercion". + */ + if (d == 0.0 && copysign(1.0, d) < 0.0) + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); + else + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + else if (PyComplex_CheckExact(op)) { + Py_complex z; + int real_negzero, imag_negzero; + /* For the complex case we must make complex(x, 0.) + different from complex(x, -0.) and complex(0., y) + different from complex(-0., y), for any x and y. + All four complex zeros must be distinguished.*/ + z = PyComplex_AsCComplex(op); + real_negzero = z.real == 0.0 && copysign(1.0, z.real) < 0.0; + imag_negzero = z.imag == 0.0 && copysign(1.0, z.imag) < 0.0; + /* use True, False and None singleton as tags for the real and imag + * sign, to make tuples different */ + if (real_negzero && imag_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_True); + } + else if (imag_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_False); + } + else if (real_negzero) { + key = PyTuple_Pack(3, Py_TYPE(op), op, Py_None); + } + else { + key = PyTuple_Pack(2, Py_TYPE(op), op); + } + } + else if (PyTuple_CheckExact(op)) { + Py_ssize_t i, len; + PyObject *tuple; + + len = PyTuple_GET_SIZE(op); + tuple = PyTuple_New(len); + if (tuple == NULL) + return NULL; + + for (i=0; i < len; i++) { + PyObject *item, *item_key; + + item = PyTuple_GET_ITEM(op, i); + item_key = _PyCode_ConstantKey(item); + if (item_key == NULL) { + Py_DECREF(tuple); + return NULL; + } + + PyTuple_SET_ITEM(tuple, i, item_key); + } + + key = PyTuple_Pack(3, Py_TYPE(op), op, tuple); + Py_DECREF(tuple); + } + else if (PyFrozenSet_CheckExact(op)) { + Py_ssize_t pos = 0; + PyObject *item; + Py_hash_t hash; + Py_ssize_t i, len; + PyObject *tuple, *set; + + len = PySet_GET_SIZE(op); + tuple = PyTuple_New(len); + if (tuple == NULL) + return NULL; + + i = 0; + while (_PySet_NextEntry(op, &pos, &item, &hash)) { + PyObject *item_key; + + item_key = _PyCode_ConstantKey(item); + if (item_key == NULL) { + Py_DECREF(tuple); + return NULL; + } + + assert(i < len); + PyTuple_SET_ITEM(tuple, i, item_key); + i++; + } + set = PyFrozenSet_New(tuple); + Py_DECREF(tuple); + if (set == NULL) + return NULL; + + key = PyTuple_Pack(3, Py_TYPE(op), op, set); + Py_DECREF(set); + return key; + } + else { + /* for other types, use the object identifier as an unique identifier + * to ensure that they are seen as unequal. */ + PyObject *obj_id = PyLong_FromVoidPtr(op); + if (obj_id == NULL) + return NULL; + + key = PyTuple_Pack(3, Py_TYPE(op), op, obj_id); + Py_DECREF(obj_id); + } + return key; +} + static PyObject * code_richcompare(PyObject *self, PyObject *other, int op) { PyCodeObject *co, *cp; int eq; + PyObject *consts1, *consts2; PyObject *res; if ((op != Py_EQ && op != Py_NE) || @@ -439,8 +563,21 @@ code_richcompare(PyObject *self, PyObject *other, int op) if (!eq) goto unequal; eq = PyObject_RichCompareBool(co->co_code, cp->co_code, Py_EQ); if (eq <= 0) goto unequal; - eq = PyObject_RichCompareBool(co->co_consts, cp->co_consts, Py_EQ); + + /* compare constants */ + consts1 = _PyCode_ConstantKey(co->co_consts); + if (!consts1) + return NULL; + consts2 = _PyCode_ConstantKey(cp->co_consts); + if (!consts2) { + Py_DECREF(consts1); + return NULL; + } + eq = PyObject_RichCompareBool(consts1, consts2, Py_EQ); + Py_DECREF(consts1); + Py_DECREF(consts2); if (eq <= 0) goto unequal; + eq = PyObject_RichCompareBool(co->co_names, cp->co_names, Py_EQ); if (eq <= 0) goto unequal; eq = PyObject_RichCompareBool(co->co_varnames, cp->co_varnames, Py_EQ); diff --git a/Python/compile.c b/Python/compile.c index f362ac6..a710e82 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -393,7 +393,7 @@ list2dict(PyObject *list) return NULL; } k = PyList_GET_ITEM(list, i); - k = PyTuple_Pack(2, k, k->ob_type); + k = _PyCode_ConstantKey(k); if (k == NULL || PyDict_SetItem(dict, k, v) < 0) { Py_XDECREF(k); Py_DECREF(v); @@ -456,7 +456,7 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return NULL; } i++; - tuple = PyTuple_Pack(2, k, k->ob_type); + tuple = _PyCode_ConstantKey(k); if (!tuple || PyDict_SetItem(dest, tuple, item) < 0) { Py_DECREF(sorted_keys); Py_DECREF(item); @@ -559,7 +559,7 @@ compiler_enter_scope(struct compiler *c, identifier name, compiler_unit_free(u); return 0; } - tuple = PyTuple_Pack(2, name, Py_TYPE(name)); + tuple = _PyCode_ConstantKey(name); if (!tuple) { compiler_unit_free(u); return 0; @@ -1105,47 +1105,8 @@ compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o) { PyObject *t, *v; Py_ssize_t arg; - double d; - - /* necessary to make sure types aren't coerced (e.g., float and complex) */ - /* _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms */ - if (PyFloat_Check(o)) { - d = PyFloat_AS_DOUBLE(o); - /* all we need is to make the tuple different in either the 0.0 - * or -0.0 case from all others, just to avoid the "coercion". - */ - if (d == 0.0 && copysign(1.0, d) < 0.0) - t = PyTuple_Pack(3, o, o->ob_type, Py_None); - else - t = PyTuple_Pack(2, o, o->ob_type); - } - else if (PyComplex_Check(o)) { - Py_complex z; - int real_negzero, imag_negzero; - /* For the complex case we must make complex(x, 0.) - different from complex(x, -0.) and complex(0., y) - different from complex(-0., y), for any x and y. - All four complex zeros must be distinguished.*/ - z = PyComplex_AsCComplex(o); - real_negzero = z.real == 0.0 && copysign(1.0, z.real) < 0.0; - imag_negzero = z.imag == 0.0 && copysign(1.0, z.imag) < 0.0; - if (real_negzero && imag_negzero) { - t = PyTuple_Pack(5, o, o->ob_type, - Py_None, Py_None, Py_None); - } - else if (imag_negzero) { - t = PyTuple_Pack(4, o, o->ob_type, Py_None, Py_None); - } - else if (real_negzero) { - t = PyTuple_Pack(3, o, o->ob_type, Py_None); - } - else { - t = PyTuple_Pack(2, o, o->ob_type); - } - } - else { - t = PyTuple_Pack(2, o, o->ob_type); - } + + t = _PyCode_ConstantKey(o); if (t == NULL) return -1; @@ -1459,7 +1420,7 @@ static int compiler_lookup_arg(PyObject *dict, PyObject *name) { PyObject *k, *v; - k = PyTuple_Pack(2, name, name->ob_type); + k = _PyCode_ConstantKey(name); if (k == NULL) return -1; v = PyDict_GetItem(dict, k); @@ -4657,9 +4618,10 @@ dict_keys_inorder(PyObject *dict, Py_ssize_t offset) return NULL; while (PyDict_Next(dict, &pos, &k, &v)) { i = PyLong_AS_LONG(v); - /* The keys of the dictionary are tuples. (see compiler_add_o) - The object we want is always first, though. */ - k = PyTuple_GET_ITEM(k, 0); + /* The keys of the dictionary are tuples. (see compiler_add_o + * and _PyCode_ConstantKey). The object we want is always second, + * though. */ + k = PyTuple_GET_ITEM(k, 1); Py_INCREF(k); assert((i - offset) < size); assert((i - offset) >= 0); -- cgit v0.12 From b02ef715a303ab09a39ef994d8f5acc4eb572376 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 14:09:55 +0100 Subject: Use Py_uintptr_t for atomic pointers Issue #26161: Use Py_uintptr_t instead of void* for atomic pointers in pyatomic.h. Use atomic_uintptr_t when is used. Using void* causes compilation warnings depending on which implementation of atomic types is used. --- Include/pyatomic.h | 6 +++--- Python/ceval_gil.h | 8 ++++---- Python/pystate.c | 47 ++++++++++++++++++++++++----------------------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/Include/pyatomic.h b/Include/pyatomic.h index 892a217..89028ef 100644 --- a/Include/pyatomic.h +++ b/Include/pyatomic.h @@ -30,7 +30,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - _Atomic void *_value; + atomic_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { @@ -61,7 +61,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - void *_value; + Py_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { @@ -98,7 +98,7 @@ typedef enum _Py_memory_order { } _Py_memory_order; typedef struct _Py_atomic_address { - void *_value; + Py_uintptr_t _value; } _Py_atomic_address; typedef struct _Py_atomic_int { diff --git a/Python/ceval_gil.h b/Python/ceval_gil.h index aafcbc2..8d38ee9 100644 --- a/Python/ceval_gil.h +++ b/Python/ceval_gil.h @@ -111,7 +111,7 @@ static _Py_atomic_int gil_locked = {-1}; static unsigned long gil_switch_number = 0; /* Last PyThreadState holding / having held the GIL. This helps us know whether anyone else was scheduled after we dropped the GIL. */ -static _Py_atomic_address gil_last_holder = {NULL}; +static _Py_atomic_address gil_last_holder = {0}; /* This condition variable allows one or several threads to wait until the GIL is released. In addition, the mutex also protects the above @@ -142,7 +142,7 @@ static void create_gil(void) #ifdef FORCE_SWITCHING COND_INIT(switch_cond); #endif - _Py_atomic_store_relaxed(&gil_last_holder, NULL); + _Py_atomic_store_relaxed(&gil_last_holder, 0); _Py_ANNOTATE_RWLOCK_CREATE(&gil_locked); _Py_atomic_store_explicit(&gil_locked, 0, _Py_memory_order_release); } @@ -178,7 +178,7 @@ static void drop_gil(PyThreadState *tstate) /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ - _Py_atomic_store_relaxed(&gil_last_holder, tstate); + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); } MUTEX_LOCK(gil_mutex); @@ -240,7 +240,7 @@ _ready: _Py_ANNOTATE_RWLOCK_ACQUIRED(&gil_locked, /*is_write=*/1); if (tstate != (PyThreadState*)_Py_atomic_load_relaxed(&gil_last_holder)) { - _Py_atomic_store_relaxed(&gil_last_holder, tstate); + _Py_atomic_store_relaxed(&gil_last_holder, (Py_uintptr_t)tstate); ++gil_switch_number; } diff --git a/Python/pystate.c b/Python/pystate.c index 6d0696d..853e5c7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3,11 +3,13 @@ #include "Python.h" -#ifndef Py_BUILD_CORE -/* ensure that PyThreadState_GET() is a macro, not an alias to - * PyThreadState_Get() */ -# error "pystate.c must be compiled with Py_BUILD_CORE defined" -#endif +#define GET_TSTATE() \ + ((PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current)) +#define SET_TSTATE(value) \ + _Py_atomic_store_relaxed(&_PyThreadState_Current, (Py_uintptr_t)(value)) +#define GET_INTERP_STATE() \ + (GET_TSTATE()->interp) + /* -------------------------------------------------------------------------- CAUTION @@ -54,7 +56,7 @@ static PyInterpreterState *interp_head = NULL; /* Assuming the current thread holds the GIL, this is the PyThreadState for the current thread. */ -_Py_atomic_address _PyThreadState_Current = {NULL}; +_Py_atomic_address _PyThreadState_Current = {0}; PyThreadFrameGetter _PyThreadState_GetFrame = NULL; #ifdef WITH_THREAD @@ -260,7 +262,7 @@ PyObject* PyState_FindModule(struct PyModuleDef* module) { Py_ssize_t index = module->m_base.m_index; - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); PyObject *res; if (module->m_slots) { return NULL; @@ -284,7 +286,7 @@ _PyState_AddModule(PyObject* module, struct PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - state = PyThreadState_GET()->interp; + state = GET_INTERP_STATE(); if (!def) return -1; if (!state->modules_by_index) { @@ -304,7 +306,7 @@ int PyState_AddModule(PyObject* module, struct PyModuleDef* def) { Py_ssize_t index; - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); if (!def) { Py_FatalError("PyState_AddModule: Module Definition is NULL"); return -1; @@ -331,7 +333,7 @@ PyState_RemoveModule(struct PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - state = PyThreadState_GET()->interp; + state = GET_INTERP_STATE(); if (index == 0) { Py_FatalError("PyState_RemoveModule: Module index invalid."); return -1; @@ -351,7 +353,7 @@ PyState_RemoveModule(struct PyModuleDef* def) void _PyState_ClearModules(void) { - PyInterpreterState *state = PyThreadState_GET()->interp; + PyInterpreterState *state = GET_INTERP_STATE(); if (state->modules_by_index) { Py_ssize_t i; for (i = 0; i < PyList_GET_SIZE(state->modules_by_index); i++) { @@ -429,7 +431,7 @@ tstate_delete_common(PyThreadState *tstate) void PyThreadState_Delete(PyThreadState *tstate) { - if (tstate == PyThreadState_GET()) + if (tstate == GET_TSTATE()) Py_FatalError("PyThreadState_Delete: tstate is still current"); #ifdef WITH_THREAD if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) @@ -443,11 +445,11 @@ PyThreadState_Delete(PyThreadState *tstate) void PyThreadState_DeleteCurrent() { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - _Py_atomic_store_relaxed(&_PyThreadState_Current, NULL); + SET_TSTATE(NULL); if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); tstate_delete_common(tstate); @@ -496,14 +498,14 @@ _PyThreadState_DeleteExcept(PyThreadState *tstate) PyThreadState * _PyThreadState_UncheckedGet(void) { - return PyThreadState_GET(); + return GET_TSTATE(); } PyThreadState * PyThreadState_Get(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) Py_FatalError("PyThreadState_Get: no current thread"); @@ -514,9 +516,9 @@ PyThreadState_Get(void) PyThreadState * PyThreadState_Swap(PyThreadState *newts) { - PyThreadState *oldts = PyThreadState_GET(); + PyThreadState *oldts = GET_TSTATE(); - _Py_atomic_store_relaxed(&_PyThreadState_Current, newts); + SET_TSTATE(newts); /* It should not be possible for more than one thread state to be used for a thread. Check this the best we can in debug builds. @@ -545,7 +547,7 @@ PyThreadState_Swap(PyThreadState *newts) PyObject * PyThreadState_GetDict(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); if (tstate == NULL) return NULL; @@ -569,8 +571,7 @@ PyThreadState_GetDict(void) int PyThreadState_SetAsyncExc(long id, PyObject *exc) { - PyThreadState *tstate = PyThreadState_GET(); - PyInterpreterState *interp = tstate->interp; + PyInterpreterState *interp = GET_INTERP_STATE(); PyThreadState *p; /* Although the GIL is held, a few C API functions can be called @@ -691,7 +692,7 @@ PyThreadState_IsCurrent(PyThreadState *tstate) { /* Must be the tstate for this thread */ assert(PyGILState_GetThisThreadState()==tstate); - return tstate == PyThreadState_GET(); + return tstate == GET_TSTATE(); } /* Internal initialization/finalization functions called by @@ -783,7 +784,7 @@ PyGILState_GetThisThreadState(void) int PyGILState_Check(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = GET_TSTATE(); return tstate && (tstate == PyGILState_GetThisThreadState()); } -- cgit v0.12 From 22756f18bca783505d864d675d984444ae5633a6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 14:16:47 +0100 Subject: Issue #25876: test_gdb: use subprocess._args_from_interpreter_flags() to test Python with more options. --- Lib/test/test_gdb.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 78fc55c..cc5aab1 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -176,6 +176,7 @@ class DebuggerTests(unittest.TestCase): args = ['--eval-command=%s' % cmd for cmd in commands] args += ["--args", sys.executable] + args.extend(subprocess._args_from_interpreter_flags()) if not import_site: # -S suppresses the default 'import site' @@ -301,7 +302,9 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of dictionaries' self.assertGdbRepr({}) self.assertGdbRepr({'foo': 'bar'}, "{'foo': 'bar'}") - self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") + # PYTHONHASHSEED is need to get the exact item order + if not sys.flags.ignore_environment: + self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, "{'douglas': 42, 'foo': 'bar'}") def test_lists(self): 'Verify the pretty-printing of lists' @@ -379,9 +382,12 @@ id(s)''') 'Verify the pretty-printing of frozensets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later") - self.assertGdbRepr(frozenset(), 'frozenset()') - self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") - self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + self.assertGdbRepr(frozenset(), "frozenset()") + self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") + self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") def test_exceptions(self): # Test a RuntimeError @@ -510,6 +516,10 @@ id(foo)''') def test_builtins_help(self): 'Ensure that the new-style class _Helper in site.py can be handled' + + if sys.flags.no_site: + self.skipTest("need site module, but -S option was used") + # (this was the issue causing tracebacks in # http://bugs.python.org/issue8032#msg100537 ) gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) -- cgit v0.12 From 5a701f03047dbf14132e447e6bb6bdfe8ea9c33e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 22 Jan 2016 15:04:27 +0100 Subject: Issue #25876: Fix also test_set() of test_gdb when -E command line is used --- Lib/test/test_gdb.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index cc5aab1..3fe15e4 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -367,9 +367,12 @@ class PrettyPrintTests(DebuggerTests): 'Verify the pretty-printing of sets' if (gdb_major_version, gdb_minor_version) < (7, 3): self.skipTest("pretty-printing of sets needs gdb 7.3 or later") - self.assertGdbRepr(set(), 'set()') - self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") - self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + self.assertGdbRepr(set(), "set()") + self.assertGdbRepr(set(['a']), "{'a'}") + # PYTHONHASHSEED is need to get the exact frozenset item order + if not sys.flags.ignore_environment: + self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") + self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") # Ensure that we handle sets containing the "dummy" key value, # which happens on deletion: -- cgit v0.12 From 849113af6ba96c36ffaaa9896c9a337eb3253b89 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:25:50 -0800 Subject: Issue #25791: Warn when __package__ != __spec__.parent. In a previous change, __spec__.parent was prioritized over __package__. That is a backwards-compatibility break, but we do eventually want __spec__ to be the ground truth for module details. So this change reverts the change in semantics and instead raises an ImportWarning when __package__ != __spec__.parent to give people time to adjust to using spec objects. --- Doc/reference/import.rst | 18 +++- Doc/whatsnew/3.6.rst | 6 +- Lib/importlib/_bootstrap.py | 12 ++- .../test_importlib/import_/test___package__.py | 33 ++++-- .../import_/test_relative_imports.py | 16 ++- Misc/NEWS | 4 +- Python/import.c | 97 ++++++++++------- Python/importlib.h | 117 +++++++++++---------- 8 files changed, 183 insertions(+), 120 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index a162851..56049ce 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -554,20 +554,30 @@ the module. details. This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules -- as defined in :pep:`366` -- - when ``__spec__`` is not defined. + relative imports for main modules, as defined in :pep:`366`. It is + expected to have the same value as ``__spec__.parent``. + + .. versionchanged:: 3.6 + The value of ``__package__`` is expected to be the same as + ``__spec__.parent``. .. attribute:: __spec__ The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. This is used primarily for - introspection and during reloading. Setting ``__spec__`` + used when importing the module. Setting ``__spec__`` appropriately applies equally to :ref:`modules initialized during interpreter startup `. The one exception is ``__main__``, where ``__spec__`` is :ref:`set to None in some cases `. + When ``__package__`` is not defined, ``__spec__.parent`` is used as + a fallback. + .. versionadded:: 3.4 + .. versionchanged:: 3.6 + ``__spec__.parent`` is used as a fallback when ``__package__`` is + not defined. + .. attribute:: __path__ If the module is a package (either regular or namespace), the module diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8064537..5872cc0 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -274,9 +274,9 @@ Changes in the Python API :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. -* When performing a relative import, ``__spec__.parent`` is used - is ``__spec__`` is defined instead of ``__package__``. - (Contributed by Rose Ames in :issue:`25791`.) +* When performing a relative import, if ``__package__`` does not compare equal + to ``__spec__.parent`` then :exc:`ImportWarning` is raised. + (Contributed by Brett Cannon in :issue:`25791`.) Changes in the C API diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 9adcf7b..f0a4e1c 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1032,11 +1032,17 @@ def _calc___package__(globals): to represent that its proper value is unknown. """ + package = globals.get('__package__') spec = globals.get('__spec__') - if spec is not None: + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn("__package__ != __spec__.parent " + f"({package!r} != {spec.parent!r})", + ImportWarning, stacklevel=3) + return package + elif spec is not None: return spec.parent - package = globals.get('__package__') - if package is None: + else: _warnings.warn("can't resolve package from __spec__ or __package__, " "falling back on __name__ and __path__", ImportWarning, stacklevel=3) diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 08e41c9..ddeeaca 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -5,6 +5,7 @@ of using the typical __path__/__name__ test). """ import unittest +import warnings from .. import util @@ -38,8 +39,8 @@ class Using__package__: with util.import_state(meta_path=[importer]): self.__import__('pkg.fake') module = self.__import__('', - globals=globals_, - fromlist=['attr'], level=2) + globals=globals_, + fromlist=['attr'], level=2) return module def test_using___package__(self): @@ -49,7 +50,10 @@ class Using__package__: def test_using___name__(self): # [__name__] - module = self.import_module({'__name__': 'pkg.fake', '__path__': []}) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({'__name__': 'pkg.fake', + '__path__': []}) self.assertEqual(module.__name__, 'pkg') def test_warn_when_using___name__(self): @@ -58,14 +62,22 @@ class Using__package__: def test_None_as___package__(self): # [None] - module = self.import_module({ - '__name__': 'pkg.fake', '__path__': [], '__package__': None }) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + module = self.import_module({ + '__name__': 'pkg.fake', '__path__': [], '__package__': None }) self.assertEqual(module.__name__, 'pkg') - def test_prefers___spec__(self): - globals = {'__spec__': FakeSpec()} - with self.assertRaises(SystemError): - self.__import__('', globals, {}, ['relimport'], 1) + def test_spec_fallback(self): + # If __package__ isn't defined, fall back on __spec__.parent. + module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) + self.assertEqual(module.__name__, 'pkg') + + def test_warn_when_package_and_spec_disagree(self): + # Raise an ImportWarning if __package__ != __spec__.parent. + with self.assertWarns(ImportWarning): + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} @@ -79,7 +91,8 @@ class Using__package__: class FakeSpec: - parent = '' + def __init__(self, parent): + self.parent = parent class Using__package__PEP302(Using__package__): diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 28bb6f7..0409f22 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -2,6 +2,8 @@ from .. import util import sys import unittest +import warnings + class RelativeImports: @@ -65,9 +67,11 @@ class RelativeImports: uncache_names.append(name[:-len('.__init__')]) with util.mock_spec(*create) as importer: with util.import_state(meta_path=[importer]): - for global_ in globals_: - with util.uncache(*uncache_names): - callback(global_) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + for global_ in globals_: + with util.uncache(*uncache_names): + callback(global_) def test_module_from_module(self): @@ -204,8 +208,10 @@ class RelativeImports: def test_relative_import_no_globals(self): # No globals for a relative import is an error. - with self.assertRaises(KeyError): - self.__import__('sys', level=1) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with self.assertRaises(KeyError): + self.__import__('sys', level=1) (Frozen_RelativeImports, diff --git a/Misc/NEWS b/Misc/NEWS index 409feae..165b428 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -26,8 +26,8 @@ Core and Builtins Python 3.5.1 to hide the exact implementation of atomic C types, to avoid compiler issues. -- Issue #25791: Trying to resolve a relative import without __spec__ or - __package__ defined now raises an ImportWarning +- Issue #25791: If __package__ != __spec__.parent or if neither __package__ or + __spec__ are defined then ImportWarning is raised. - Issue #25731: Fix set and deleting __new__ on a class. diff --git a/Python/import.c b/Python/import.c index c1071c0..22f9d21 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1415,60 +1415,79 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error; } else if (level > 0) { + package = _PyDict_GetItemId(globals, &PyId___package__); spec = _PyDict_GetItemId(globals, &PyId___spec__); - if (spec != NULL) { - package = PyObject_GetAttrString(spec, "parent"); - } + if (package != NULL && package != Py_None) { + Py_INCREF(package); if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; + } + else if (spec != NULL) { + int equal; + PyObject *parent = PyObject_GetAttrString(spec, "parent"); + if (parent == NULL) { + goto error; + } + + equal = PyObject_RichCompareBool(package, parent, Py_EQ); + Py_DECREF(parent); + if (equal < 0) { + goto error; + } + else if (equal == 0) { + if (PyErr_WarnEx(PyExc_ImportWarning, + "__package__ != __spec__.parent", 1) < 0) { + goto error; + } + } + } + } + else if (spec != NULL) { + package = PyObject_GetAttrString(spec, "parent"); + if (package == NULL) { + goto error; + } + else if (!PyUnicode_Check(package)) { PyErr_SetString(PyExc_TypeError, "__spec__.parent must be a string"); goto error; } } else { - package = _PyDict_GetItemId(globals, &PyId___package__); - if (package != NULL && package != Py_None) { - Py_INCREF(package); - if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "package must be a string"); - goto error; - } + if (PyErr_WarnEx(PyExc_ImportWarning, + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", 1) < 0) { + goto error; } - else { - if (PyErr_WarnEx(PyExc_ImportWarning, - "can't resolve package from __spec__ or __package__, " - "falling back on __name__ and __path__", 1) < 0) { - goto error; - } - package = _PyDict_GetItemId(globals, &PyId___name__); - if (package == NULL) { - PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); - goto error; - } + package = _PyDict_GetItemId(globals, &PyId___name__); + if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); + goto error; + } - Py_INCREF(package); - if (!PyUnicode_Check(package)) { - PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + goto error; + } + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { goto error; } - - if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { - PyObject *partition = NULL; - PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); - if (borrowed_dot == NULL) { - goto error; - } - partition = PyUnicode_RPartition(package, borrowed_dot); - Py_DECREF(package); - if (partition == NULL) { - goto error; - } - package = PyTuple_GET_ITEM(partition, 0); - Py_INCREF(package); - Py_DECREF(partition); + partition = PyUnicode_RPartition(package, borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; } + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); } } diff --git a/Python/importlib.h b/Python/importlib.h index 709c5fd..e4ef835 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1787,43 +1787,52 @@ const unsigned char _Py_M__importlib[] = { 115,116,228,3,0,0,115,34,0,0,0,0,10,15,1,12, 1,12,1,13,1,15,1,16,1,13,1,15,1,21,1,3, 1,17,1,18,4,21,1,15,1,3,1,26,1,114,195,0, - 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,5, - 0,0,0,67,0,0,0,115,128,0,0,0,124,0,0,106, - 0,0,100,1,0,131,1,0,125,1,0,124,1,0,100,2, - 0,107,9,0,114,34,0,124,1,0,106,1,0,83,124,0, - 0,106,0,0,100,3,0,131,1,0,125,2,0,124,2,0, - 100,2,0,107,8,0,114,124,0,116,2,0,106,3,0,100, - 4,0,116,4,0,100,5,0,100,6,0,131,2,1,1,124, - 0,0,100,7,0,25,125,2,0,100,8,0,124,0,0,107, - 7,0,114,124,0,124,2,0,106,5,0,100,9,0,131,1, - 0,100,10,0,25,125,2,0,124,2,0,83,41,11,122,167, - 67,97,108,99,117,108,97,116,101,32,119,104,97,116,32,95, - 95,112,97,99,107,97,103,101,95,95,32,115,104,111,117,108, - 100,32,98,101,46,10,10,32,32,32,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,115,32,110,111,116,32,103,117, - 97,114,97,110,116,101,101,100,32,116,111,32,98,101,32,100, - 101,102,105,110,101,100,32,111,114,32,99,111,117,108,100,32, - 98,101,32,115,101,116,32,116,111,32,78,111,110,101,10,32, - 32,32,32,116,111,32,114,101,112,114,101,115,101,110,116,32, - 116,104,97,116,32,105,116,115,32,112,114,111,112,101,114,32, - 118,97,108,117,101,32,105,115,32,117,110,107,110,111,119,110, - 46,10,10,32,32,32,32,114,95,0,0,0,78,114,134,0, - 0,0,122,89,99,97,110,39,116,32,114,101,115,111,108,118, - 101,32,112,97,99,107,97,103,101,32,102,114,111,109,32,95, - 95,115,112,101,99,95,95,32,111,114,32,95,95,112,97,99, - 107,97,103,101,95,95,44,32,102,97,108,108,105,110,103,32, - 98,97,99,107,32,111,110,32,95,95,110,97,109,101,95,95, - 32,97,110,100,32,95,95,112,97,116,104,95,95,114,140,0, - 0,0,233,3,0,0,0,114,1,0,0,0,114,131,0,0, - 0,114,121,0,0,0,114,33,0,0,0,41,6,114,42,0, - 0,0,114,123,0,0,0,114,142,0,0,0,114,143,0,0, - 0,114,176,0,0,0,114,122,0,0,0,41,3,218,7,103, - 108,111,98,97,108,115,114,88,0,0,0,114,170,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,218, - 17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,101, - 95,95,4,4,0,0,115,22,0,0,0,0,7,15,1,12, - 1,7,1,15,1,12,1,9,2,13,1,10,1,12,1,19, - 1,114,198,0,0,0,99,5,0,0,0,0,0,0,0,9, + 0,0,99,1,0,0,0,0,0,0,0,3,0,0,0,7, + 0,0,0,67,0,0,0,115,214,0,0,0,124,0,0,106, + 0,0,100,1,0,131,1,0,125,1,0,124,0,0,106,0, + 0,100,2,0,131,1,0,125,2,0,124,1,0,100,3,0, + 107,9,0,114,128,0,124,2,0,100,3,0,107,9,0,114, + 124,0,124,1,0,124,2,0,106,1,0,107,3,0,114,124, + 0,116,2,0,106,3,0,100,4,0,106,4,0,100,5,0, + 124,1,0,155,2,0,100,6,0,124,2,0,106,1,0,155, + 2,0,100,7,0,103,5,0,131,1,0,116,5,0,100,8, + 0,100,9,0,131,2,1,1,124,1,0,83,124,2,0,100, + 3,0,107,9,0,114,147,0,124,2,0,106,1,0,83,116, + 2,0,106,3,0,100,10,0,116,5,0,100,8,0,100,9, + 0,131,2,1,1,124,0,0,100,11,0,25,125,1,0,100, + 12,0,124,0,0,107,7,0,114,210,0,124,1,0,106,6, + 0,100,13,0,131,1,0,100,14,0,25,125,1,0,124,1, + 0,83,41,15,122,167,67,97,108,99,117,108,97,116,101,32, + 119,104,97,116,32,95,95,112,97,99,107,97,103,101,95,95, + 32,115,104,111,117,108,100,32,98,101,46,10,10,32,32,32, + 32,95,95,112,97,99,107,97,103,101,95,95,32,105,115,32, + 110,111,116,32,103,117,97,114,97,110,116,101,101,100,32,116, + 111,32,98,101,32,100,101,102,105,110,101,100,32,111,114,32, + 99,111,117,108,100,32,98,101,32,115,101,116,32,116,111,32, + 78,111,110,101,10,32,32,32,32,116,111,32,114,101,112,114, + 101,115,101,110,116,32,116,104,97,116,32,105,116,115,32,112, + 114,111,112,101,114,32,118,97,108,117,101,32,105,115,32,117, + 110,107,110,111,119,110,46,10,10,32,32,32,32,114,134,0, + 0,0,114,95,0,0,0,78,218,0,122,32,95,95,112,97, + 99,107,97,103,101,95,95,32,33,61,32,95,95,115,112,101, + 99,95,95,46,112,97,114,101,110,116,32,40,122,4,32,33, + 61,32,250,1,41,114,140,0,0,0,233,3,0,0,0,122, + 89,99,97,110,39,116,32,114,101,115,111,108,118,101,32,112, + 97,99,107,97,103,101,32,102,114,111,109,32,95,95,115,112, + 101,99,95,95,32,111,114,32,95,95,112,97,99,107,97,103, + 101,95,95,44,32,102,97,108,108,105,110,103,32,98,97,99, + 107,32,111,110,32,95,95,110,97,109,101,95,95,32,97,110, + 100,32,95,95,112,97,116,104,95,95,114,1,0,0,0,114, + 131,0,0,0,114,121,0,0,0,114,33,0,0,0,41,7, + 114,42,0,0,0,114,123,0,0,0,114,142,0,0,0,114, + 143,0,0,0,114,115,0,0,0,114,176,0,0,0,114,122, + 0,0,0,41,3,218,7,103,108,111,98,97,108,115,114,170, + 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,17,95,99,97,108,99,95,95, + 95,112,97,99,107,97,103,101,95,95,4,4,0,0,115,30, + 0,0,0,0,7,15,1,15,1,12,1,27,1,42,2,13, + 1,4,1,12,1,7,2,9,2,13,1,10,1,12,1,19, + 1,114,200,0,0,0,99,5,0,0,0,0,0,0,0,9, 0,0,0,5,0,0,0,67,0,0,0,115,227,0,0,0, 124,4,0,100,1,0,107,2,0,114,27,0,116,0,0,124, 0,0,131,1,0,125,5,0,110,54,0,124,1,0,100,2, @@ -1870,17 +1879,17 @@ const unsigned char _Py_M__importlib[] = { 111,117,108,100,32,104,97,118,101,32,97,32,39,108,101,118, 101,108,39,32,111,102,32,50,41,46,10,10,32,32,32,32, 114,33,0,0,0,78,114,121,0,0,0,41,8,114,187,0, - 0,0,114,198,0,0,0,218,9,112,97,114,116,105,116,105, + 0,0,114,200,0,0,0,218,9,112,97,114,116,105,116,105, 111,110,114,168,0,0,0,114,14,0,0,0,114,21,0,0, 0,114,1,0,0,0,114,195,0,0,0,41,9,114,15,0, - 0,0,114,197,0,0,0,218,6,108,111,99,97,108,115,114, + 0,0,114,199,0,0,0,218,6,108,111,99,97,108,115,114, 193,0,0,0,114,171,0,0,0,114,89,0,0,0,90,8, 103,108,111,98,97,108,115,95,114,170,0,0,0,90,7,99, 117,116,95,111,102,102,114,10,0,0,0,114,10,0,0,0, 114,11,0,0,0,218,10,95,95,105,109,112,111,114,116,95, - 95,25,4,0,0,115,26,0,0,0,0,11,12,1,15,2, + 95,31,4,0,0,115,26,0,0,0,0,11,12,1,15,2, 24,1,12,1,18,1,6,3,12,1,23,1,6,1,4,4, - 35,3,40,2,114,201,0,0,0,99,1,0,0,0,0,0, + 35,3,40,2,114,203,0,0,0,99,1,0,0,0,0,0, 0,0,2,0,0,0,3,0,0,0,67,0,0,0,115,53, 0,0,0,116,0,0,106,1,0,124,0,0,131,1,0,125, 1,0,124,1,0,100,0,0,107,8,0,114,43,0,116,2, @@ -1891,8 +1900,8 @@ const unsigned char _Py_M__importlib[] = { 0,0,114,77,0,0,0,114,150,0,0,0,41,2,114,15, 0,0,0,114,88,0,0,0,114,10,0,0,0,114,10,0, 0,0,114,11,0,0,0,218,18,95,98,117,105,108,116,105, - 110,95,102,114,111,109,95,110,97,109,101,60,4,0,0,115, - 8,0,0,0,0,1,15,1,12,1,16,1,114,202,0,0, + 110,95,102,114,111,109,95,110,97,109,101,66,4,0,0,115, + 8,0,0,0,0,1,15,1,12,1,16,1,114,204,0,0, 0,99,2,0,0,0,0,0,0,0,12,0,0,0,12,0, 0,0,67,0,0,0,115,74,1,0,0,124,1,0,97,0, 0,124,0,0,97,1,0,116,2,0,116,1,0,131,1,0, @@ -1937,7 +1946,7 @@ const unsigned char _Py_M__importlib[] = { 0,114,21,0,0,0,218,5,105,116,101,109,115,114,178,0, 0,0,114,76,0,0,0,114,151,0,0,0,114,82,0,0, 0,114,161,0,0,0,114,132,0,0,0,114,137,0,0,0, - 114,1,0,0,0,114,202,0,0,0,114,5,0,0,0,114, + 114,1,0,0,0,114,204,0,0,0,114,5,0,0,0,114, 77,0,0,0,41,12,218,10,115,121,115,95,109,111,100,117, 108,101,218,11,95,105,109,112,95,109,111,100,117,108,101,90, 11,109,111,100,117,108,101,95,116,121,112,101,114,15,0,0, @@ -1948,10 +1957,10 @@ const unsigned char _Py_M__importlib[] = { 101,97,100,95,109,111,100,117,108,101,90,14,119,101,97,107, 114,101,102,95,109,111,100,117,108,101,114,10,0,0,0,114, 10,0,0,0,114,11,0,0,0,218,6,95,115,101,116,117, - 112,67,4,0,0,115,50,0,0,0,0,9,6,1,6,3, + 112,73,4,0,0,115,50,0,0,0,0,9,6,1,6,3, 12,1,28,1,15,1,15,1,9,1,15,1,9,2,3,1, 15,1,17,3,13,1,13,1,15,1,15,2,13,1,20,3, - 3,1,16,1,13,2,11,1,16,3,12,1,114,206,0,0, + 3,1,16,1,13,2,11,1,16,3,12,1,114,208,0,0, 0,99,2,0,0,0,0,0,0,0,3,0,0,0,3,0, 0,0,67,0,0,0,115,87,0,0,0,116,0,0,124,0, 0,124,1,0,131,2,0,1,116,1,0,106,2,0,106,3, @@ -1963,15 +1972,15 @@ const unsigned char _Py_M__importlib[] = { 112,111,114,116,108,105,98,32,97,115,32,116,104,101,32,105, 109,112,108,101,109,101,110,116,97,116,105,111,110,32,111,102, 32,105,109,112,111,114,116,46,114,33,0,0,0,78,41,11, - 114,206,0,0,0,114,14,0,0,0,114,175,0,0,0,114, + 114,208,0,0,0,114,14,0,0,0,114,175,0,0,0,114, 113,0,0,0,114,151,0,0,0,114,161,0,0,0,218,26, 95,102,114,111,122,101,110,95,105,109,112,111,114,116,108,105, 98,95,101,120,116,101,114,110,97,108,114,119,0,0,0,218, 8,95,105,110,115,116,97,108,108,114,21,0,0,0,114,1, - 0,0,0,41,3,114,204,0,0,0,114,205,0,0,0,114, - 207,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,114,208,0,0,0,114,4,0,0,115,12,0,0, - 0,0,2,13,2,16,1,16,3,12,1,6,1,114,208,0, + 0,0,0,41,3,114,206,0,0,0,114,207,0,0,0,114, + 209,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,114,210,0,0,0,120,4,0,0,115,12,0,0, + 0,0,2,13,2,16,1,16,3,12,1,6,1,114,210,0, 0,0,41,51,114,3,0,0,0,114,119,0,0,0,114,12, 0,0,0,114,16,0,0,0,114,17,0,0,0,114,59,0, 0,0,114,41,0,0,0,114,48,0,0,0,114,31,0,0, @@ -1987,8 +1996,8 @@ const unsigned char _Py_M__importlib[] = { 0,114,172,0,0,0,114,174,0,0,0,114,177,0,0,0, 114,182,0,0,0,114,192,0,0,0,114,183,0,0,0,114, 185,0,0,0,114,186,0,0,0,114,187,0,0,0,114,195, - 0,0,0,114,198,0,0,0,114,201,0,0,0,114,202,0, - 0,0,114,206,0,0,0,114,208,0,0,0,114,10,0,0, + 0,0,0,114,200,0,0,0,114,203,0,0,0,114,204,0, + 0,0,114,208,0,0,0,114,210,0,0,0,114,10,0,0, 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0, 218,8,60,109,111,100,117,108,101,62,8,0,0,0,115,96, 0,0,0,6,17,6,2,12,8,12,4,19,20,6,2,6, @@ -1996,6 +2005,6 @@ const unsigned char _Py_M__importlib[] = { 8,12,11,12,12,12,16,12,36,19,27,19,101,24,26,9, 3,18,45,18,60,12,18,12,17,12,25,12,29,12,23,12, 16,19,73,19,77,19,13,12,9,12,9,15,40,12,17,6, - 1,10,2,12,27,12,6,18,24,12,32,12,21,24,35,12, + 1,10,2,12,27,12,6,18,24,12,32,12,27,24,35,12, 7,12,47, }; -- cgit v0.12 From b0db3718d44d4ef8aa257a166299bc35f0e08b87 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:26:56 -0800 Subject: whitespace cleanup --- Lib/test/test_importlib/import_/test___package__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index ddeeaca..7f64548 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -76,8 +76,8 @@ class Using__package__: def test_warn_when_package_and_spec_disagree(self): # Raise an ImportWarning if __package__ != __spec__.parent. with self.assertWarns(ImportWarning): - self.import_module({'__package__': 'pkg.fake', - '__spec__': FakeSpec('pkg.fakefake')}) + self.import_module({'__package__': 'pkg.fake', + '__spec__': FakeSpec('pkg.fakefake')}) def test_bad__package__(self): globals = {'__package__': ''} -- cgit v0.12 From 4b18dd339a9919e6f5fa3485c8b871ed19d9e391 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 15:55:56 -0800 Subject: Issue #25234: Skip test_eintr.test_open() under OS X to avoid hanging --- Lib/test/eintrdata/eintr_tester.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index e1ed3da..1cb86e5 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -341,6 +341,7 @@ class SocketEINTRTest(EINTRBaseTest): self._test_open("fp = open(path, 'r')\nfp.close()", self.python_open) + @unittest.skipIf(sys.platform == 'darwin', "hangs under OS X; see issue #25234") def os_open(self, path): fd = os.open(path, os.O_WRONLY) os.close(fd) -- cgit v0.12 From 9fa812668faf0d2d7839845e1a0da19b87bdbc29 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 22 Jan 2016 16:39:02 -0800 Subject: Issue #18018: Raise an ImportError if a relative import is attempted with no known parent package. Previously SystemError was raised if the parent package didn't exist (e.g., __package__ was set to ''). Thanks to Florent Xicluna and Yongzhi Pan for reporting the issue. --- Doc/whatsnew/3.6.rst | 4 ++++ Lib/test/test_importlib/import_/test_relative_imports.py | 5 +++++ Misc/NEWS | 3 +++ Python/import.c | 11 ++++++++--- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 5872cc0..158dad3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -278,6 +278,10 @@ Changes in the Python API to ``__spec__.parent`` then :exc:`ImportWarning` is raised. (Contributed by Brett Cannon in :issue:`25791`.) +* When a relative import is performed and no parent package is known, then + :exc:`ImportError` will be raised. Previously, :exc:`SystemError` could be + raised. (Contribute by Brett Cannon in :issue:`18018`.) + Changes in the C API -------------------- diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index 0409f22..1cad6b3 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -213,6 +213,11 @@ class RelativeImports: with self.assertRaises(KeyError): self.__import__('sys', level=1) + def test_relative_import_no_package(self): + with self.assertRaises(ImportError): + self.__import__('a', {'__package__': '', '__spec__': None}, + level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Misc/NEWS b/Misc/NEWS index 165b428..676f2ca 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #18018: Import raises ImportError instead of SystemError if a relative + import is attempted without a known parent package. + - Issue #25843: When compiling code, don't merge constants if they are equal but have a different types. For example, ``f1, f2 = lambda: 1, lambda: 1.0`` is now correctly compiled to two different functions: ``f1()`` returns ``1`` diff --git a/Python/import.c b/Python/import.c index 22f9d21..8ad5b4c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1424,7 +1424,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyErr_SetString(PyExc_TypeError, "package must be a string"); goto error; } - else if (spec != NULL) { + else if (spec != NULL && spec != Py_None) { int equal; PyObject *parent = PyObject_GetAttrString(spec, "parent"); if (parent == NULL) { @@ -1444,7 +1444,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } } - else if (spec != NULL) { + else if (spec != NULL && spec != Py_None) { package = PyObject_GetAttrString(spec, "parent"); if (package == NULL) { goto error; @@ -1491,7 +1491,12 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } - if (PyDict_GetItem(interp->modules, package) == NULL) { + if (PyUnicode_CompareWithASCIIString(package, "") == 0) { + PyErr_SetString(PyExc_ImportError, + "attempted relative import with no known parent package"); + goto error; + } + else if (PyDict_GetItem(interp->modules, package) == NULL) { PyErr_Format(PyExc_SystemError, "Parent module %R not loaded, cannot perform relative " "import", package); -- cgit v0.12 From 843a1fb1e78bf2d374daccaaa5c616f63afdf053 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:29:02 +0100 Subject: test_gc: remove unused imports --- Lib/test/test_gc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 1f0867d..872ca62 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -682,7 +682,6 @@ class GCTests(unittest.TestCase): # Create a reference cycle through the __main__ module and check # it gets collected at interpreter shutdown. code = """if 1: - import weakref class C: def __del__(self): print('__del__ called') @@ -696,7 +695,6 @@ class GCTests(unittest.TestCase): # Same as above, but with a non-__main__ module. with temp_dir() as script_dir: module = """if 1: - import weakref class C: def __del__(self): print('__del__ called') -- cgit v0.12 From 5ebe2c89fe47501db64e1a82ac6a3bcbd21f6e70 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 13:52:05 +0100 Subject: Cleanup test_dict * Write one import per line * Sort imports by name * Add an empty line: 2 empty lines between code blocks at the module level (PEP 8) --- Lib/test/test_dict.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3b42414..1049be5 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1,10 +1,12 @@ -import unittest -from test import support - -import collections, random, string +import collections import collections.abc -import gc, weakref +import gc import pickle +import random +import string +import unittest +import weakref +from test import support class DictTest(unittest.TestCase): @@ -963,5 +965,6 @@ class Dict(dict): class SubclassMappingTests(mapping_tests.BasicTestMappingProtocol): type2test = Dict + if __name__ == "__main__": unittest.main() -- cgit v0.12 From 1aa78938b0d019b7c9cbb153c2f35709ee01a19a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 23 Jan 2016 14:15:48 +0100 Subject: Issue #26146: marshal.loads() now uses the empty frozenset singleton --- Lib/test/test_marshal.py | 7 +++++ Python/marshal.c | 69 ++++++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index c7def9a..b378ffe 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -135,6 +135,13 @@ class ContainerTestCase(unittest.TestCase, HelperMixin): for constructor in (set, frozenset): self.helper(constructor(self.d.keys())) + @support.cpython_only + def test_empty_frozenset_singleton(self): + # marshal.loads() must reuse the empty frozenset singleton + obj = frozenset() + obj2 = marshal.loads(marshal.dumps(obj)) + self.assertIs(obj2, obj) + class BufferTestCase(unittest.TestCase, HelperMixin): diff --git a/Python/marshal.c b/Python/marshal.c index 589eb80..7a4b9d2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1266,41 +1266,52 @@ r_object(RFILE *p) PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)"); break; } - v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); - if (type == TYPE_SET) { - R_REF(v); - } else { - /* must use delayed registration of frozensets because they must - * be init with a refcount of 1 - */ - idx = r_ref_reserve(flag, p); - if (idx < 0) - Py_CLEAR(v); /* signal error */ - } - if (v == NULL) - break; - for (i = 0; i < n; i++) { - v2 = r_object(p); - if ( v2 == NULL ) { - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "NULL object in marshal data for set"); - Py_DECREF(v); - v = NULL; + if (n == 0 && type == TYPE_FROZENSET) { + /* call frozenset() to get the empty frozenset singleton */ + v = PyObject_CallFunction((PyObject*)&PyFrozenSet_Type, NULL); + if (v == NULL) break; + R_REF(v); + retval = v; + } + else { + v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL); + if (type == TYPE_SET) { + R_REF(v); + } else { + /* must use delayed registration of frozensets because they must + * be init with a refcount of 1 + */ + idx = r_ref_reserve(flag, p); + if (idx < 0) + Py_CLEAR(v); /* signal error */ } - if (PySet_Add(v, v2) == -1) { - Py_DECREF(v); - Py_DECREF(v2); - v = NULL; + if (v == NULL) break; + + for (i = 0; i < n; i++) { + v2 = r_object(p); + if ( v2 == NULL ) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, + "NULL object in marshal data for set"); + Py_DECREF(v); + v = NULL; + break; + } + if (PySet_Add(v, v2) == -1) { + Py_DECREF(v); + Py_DECREF(v2); + v = NULL; + break; + } + Py_DECREF(v2); } - Py_DECREF(v2); + if (type != TYPE_SET) + v = r_ref_insert(v, idx, flag, p); + retval = v; } - if (type != TYPE_SET) - v = r_ref_insert(v, idx, flag, p); - retval = v; break; case TYPE_CODE: -- cgit v0.12 From d84ec225bdf88a8ad54c57b78beb6c32ae9fffde Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 09:12:06 -0800 Subject: Miscellaneous refactorings * Add comment to the maxlen structure entry about the meaning of maxlen == -1 * Factor-out code common to deque_append(left) and deque_extend(left) * Factor inner-loop in deque_clear() to use only 1 test per loop instead of 2 * Tighten inner-loops for deque_item() and deque_ass_item() so that the compiler can combine the decrement and test into a single step. --- Modules/_collectionsmodule.c | 123 ++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 65 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index fbf07a8..3ab987d 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -81,7 +81,7 @@ typedef struct { Py_ssize_t leftindex; /* 0 <= leftindex < BLOCKLEN */ Py_ssize_t rightindex; /* 0 <= rightindex < BLOCKLEN */ size_t state; /* incremented whenever the indices move */ - Py_ssize_t maxlen; + Py_ssize_t maxlen; /* maxlen is -1 for unbounded deques */ PyObject *weakreflist; } dequeobject; @@ -264,13 +264,13 @@ PyDoc_STRVAR(popleft_doc, "Remove and return the leftmost element."); #define NEEDS_TRIM(deque, maxlen) ((size_t)(maxlen) < (size_t)(Py_SIZE(deque))) -static PyObject * -deque_append(dequeobject *deque, PyObject *item) +int +deque_append_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->rightindex == BLOCKLEN - 1) { block *b = newblock(); if (b == NULL) - return NULL; + return -1; b->leftlink = deque->rightblock; CHECK_END(deque->rightblock->rightlink); deque->rightblock->rightlink = b; @@ -279,27 +279,35 @@ deque_append(dequeobject *deque, PyObject *item) deque->rightindex = -1; } Py_SIZE(deque)++; - Py_INCREF(item); deque->rightindex++; deque->rightblock->data[deque->rightindex] = item; - if (NEEDS_TRIM(deque, deque->maxlen)) { + if (NEEDS_TRIM(deque, maxlen)) { PyObject *olditem = deque_popleft(deque, NULL); Py_DECREF(olditem); } else { deque->state++; } + return 0; +} + +static PyObject * +deque_append(dequeobject *deque, PyObject *item) +{ + Py_INCREF(item); + if (deque_append_internal(deque, item, deque->maxlen) < 0) + return NULL; Py_RETURN_NONE; } PyDoc_STRVAR(append_doc, "Add an element to the right side of the deque."); -static PyObject * -deque_appendleft(dequeobject *deque, PyObject *item) +int +deque_appendleft_internal(dequeobject *deque, PyObject *item, Py_ssize_t maxlen) { if (deque->leftindex == 0) { block *b = newblock(); if (b == NULL) - return NULL; + return -1; b->rightlink = deque->leftblock; CHECK_END(deque->leftblock->leftlink); deque->leftblock->leftlink = b; @@ -308,7 +316,6 @@ deque_appendleft(dequeobject *deque, PyObject *item) deque->leftindex = BLOCKLEN; } Py_SIZE(deque)++; - Py_INCREF(item); deque->leftindex--; deque->leftblock->data[deque->leftindex] = item; if (NEEDS_TRIM(deque, deque->maxlen)) { @@ -317,6 +324,15 @@ deque_appendleft(dequeobject *deque, PyObject *item) } else { deque->state++; } + return 0; +} + +static PyObject * +deque_appendleft(dequeobject *deque, PyObject *item) +{ + Py_INCREF(item); + if (deque_appendleft_internal(deque, item, deque->maxlen) < 0) + return NULL; Py_RETURN_NONE; } @@ -387,28 +403,10 @@ deque_extend(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque->rightindex == BLOCKLEN - 1) { - block *b = newblock(); - if (b == NULL) { - Py_DECREF(item); - Py_DECREF(it); - return NULL; - } - b->leftlink = deque->rightblock; - CHECK_END(deque->rightblock->rightlink); - deque->rightblock->rightlink = b; - deque->rightblock = b; - MARK_END(b->rightlink); - deque->rightindex = -1; - } - Py_SIZE(deque)++; - deque->rightindex++; - deque->rightblock->data[deque->rightindex] = item; - if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_popleft(deque, NULL); - Py_DECREF(olditem); - } else { - deque->state++; + if (deque_append_internal(deque, item, maxlen) < 0) { + Py_DECREF(item); + Py_DECREF(it); + return NULL; } } return finalize_iterator(it); @@ -452,28 +450,10 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { - if (deque->leftindex == 0) { - block *b = newblock(); - if (b == NULL) { - Py_DECREF(item); - Py_DECREF(it); - return NULL; - } - b->rightlink = deque->leftblock; - CHECK_END(deque->leftblock->leftlink); - deque->leftblock->leftlink = b; - deque->leftblock = b; - MARK_END(b->leftlink); - deque->leftindex = BLOCKLEN; - } - Py_SIZE(deque)++; - deque->leftindex--; - deque->leftblock->data[deque->leftindex] = item; - if (NEEDS_TRIM(deque, maxlen)) { - PyObject *olditem = deque_pop(deque, NULL); - Py_DECREF(olditem); - } else { - deque->state++; + if (deque_appendleft_internal(deque, item, maxlen) < 0) { + Py_DECREF(item); + Py_DECREF(it); + return NULL; } } return finalize_iterator(it); @@ -565,8 +545,9 @@ deque_clear(dequeobject *deque) block *prevblock; block *leftblock; Py_ssize_t leftindex; - Py_ssize_t n; + Py_ssize_t n, m; PyObject *item; + PyObject **itemptr, **limit; if (Py_SIZE(deque) == 0) return; @@ -608,17 +589,25 @@ deque_clear(dequeobject *deque) /* Now the old size, leftblock, and leftindex are disconnected from the empty deque and we can use them to decref the pointers. */ - while (n--) { - item = leftblock->data[leftindex]; - Py_DECREF(item); - leftindex++; - if (leftindex == BLOCKLEN && n) { + itemptr = &leftblock->data[leftindex]; + m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; + limit = &leftblock->data[leftindex + m]; + n -= m; + while (1) { + if (itemptr == limit) { + if (n == 0) + break; CHECK_NOT_END(leftblock->rightlink); prevblock = leftblock; leftblock = leftblock->rightlink; - leftindex = 0; + itemptr = leftblock->data; + m = (n > BLOCKLEN) ? BLOCKLEN : n; + limit = &leftblock->data[m]; + n -= m; freeblock(prevblock); } + item = *(itemptr++); + Py_DECREF(item); } CHECK_END(leftblock->rightlink); freeblock(leftblock); @@ -1185,14 +1174,16 @@ deque_item(dequeobject *deque, Py_ssize_t i) i = (Py_ssize_t)((size_t) i % BLOCKLEN); if (index < (Py_SIZE(deque) >> 1)) { b = deque->leftblock; - while (n--) + n++; + while (--n) b = b->rightlink; } else { n = (Py_ssize_t)( ((size_t)(deque->leftindex + Py_SIZE(deque) - 1)) / BLOCKLEN - n); b = deque->rightblock; - while (n--) + n++; + while (--n) b = b->leftlink; } } @@ -1235,14 +1226,16 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) i = (Py_ssize_t)((size_t) i % BLOCKLEN); if (index <= halflen) { b = deque->leftblock; - while (n--) + n++; + while (--n) b = b->rightlink; } else { n = (Py_ssize_t)( ((size_t)(deque->leftindex + Py_SIZE(deque) - 1)) / BLOCKLEN - n); b = deque->rightblock; - while (n--) + n++; + while (--n) b = b->leftlink; } Py_INCREF(v); -- cgit v0.12 From 165eee214bc388eb588db33385ca49ddbb305565 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 11:32:07 -0800 Subject: Convert two other post-decrement while-loops to pre-decrements for consistency and for better code generation. --- Modules/_collectionsmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 3ab987d..cc9e4e8 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -937,7 +937,8 @@ deque_count(dequeobject *deque, PyObject *v) PyObject *item; int cmp; - while (n--) { + n++; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); @@ -974,7 +975,8 @@ deque_contains(dequeobject *deque, PyObject *v) PyObject *item; int cmp; - while (n--) { + n++; + while (--n) { CHECK_NOT_END(b); item = b->data[index]; cmp = PyObject_RichCompareBool(item, v, Py_EQ); -- cgit v0.12 From 306d6b1ea6bf2582b9284be2fd27275abbade3e1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 12:40:42 -0800 Subject: Convert another post-decrement while-loop to pre-decrement for consistency and better generated code (on both GCC and CLang). --- Modules/_collectionsmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index cc9e4e8..b77ea65 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -895,7 +895,8 @@ deque_reverse(dequeobject *deque, PyObject *unused) Py_ssize_t n = Py_SIZE(deque) >> 1; PyObject *tmp; - while (n-- > 0) { + n++; + while (--n) { /* Validate that pointers haven't met in the middle */ assert(leftblock != rightblock || leftindex < rightindex); CHECK_NOT_END(leftblock); -- cgit v0.12 From 1ce6885f8a5bc2cca1fb03fa0b0bd953407f3c25 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 24 Jan 2016 22:15:20 -0800 Subject: Fix compiler warning about obviously unreachable code. --- Modules/socketmodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7ab534e..a6143c6 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -5411,10 +5411,6 @@ socket_inet_ntop(PyObject *self, PyObject *args) } else { return PyUnicode_FromString(retval); } - - /* NOTREACHED */ - PyErr_SetString(PyExc_RuntimeError, "invalid handling of inet_ntop"); - return NULL; } #elif defined(MS_WINDOWS) -- cgit v0.12 From 0dceb918668399bdcbe27d1c59d01c4c9228c1a6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Jan 2016 10:33:30 -0800 Subject: Tests versus zero are more compact than tests versus -1. --- Objects/listobject.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index fcc21cb..bcf587c 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -248,7 +248,7 @@ ins1(PyListObject *self, Py_ssize_t where, PyObject *v) return -1; } - if (list_resize(self, n+1) == -1) + if (list_resize(self, n+1) < 0) return -1; if (where < 0) { @@ -288,7 +288,7 @@ app1(PyListObject *self, PyObject *v) return -1; } - if (list_resize(self, n+1) == -1) + if (list_resize(self, n+1) < 0) return -1; Py_INCREF(v); @@ -708,7 +708,7 @@ list_inplace_repeat(PyListObject *self, Py_ssize_t n) return PyErr_NoMemory(); } - if (list_resize(self, size*n) == -1) + if (list_resize(self, size*n) < 0) return NULL; p = size; @@ -798,7 +798,7 @@ listextend(PyListObject *self, PyObject *b) Py_RETURN_NONE; } m = Py_SIZE(self); - if (list_resize(self, m + n) == -1) { + if (list_resize(self, m + n) < 0) { Py_DECREF(b); return NULL; } @@ -826,7 +826,7 @@ listextend(PyListObject *self, PyObject *b) /* Guess a result list size. */ n = PyObject_LengthHint(b, 8); - if (n == -1) { + if (n < 0) { Py_DECREF(it); return NULL; } @@ -834,7 +834,7 @@ listextend(PyListObject *self, PyObject *b) mn = m + n; if (mn >= m) { /* Make room. */ - if (list_resize(self, mn) == -1) + if (list_resize(self, mn) < 0) goto error; /* Make the list sane again. */ Py_SIZE(self) = m; -- cgit v0.12 From f2c1aa1661edb3e14ff8b7b9995f93e303c8acbb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 26 Jan 2016 00:40:57 +0100 Subject: Add ast.Constant Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by external AST optimizers, but the compiler does not emit directly such node. An optimizer can replace the following AST nodes with ast.Constant: * ast.NameConstant: None, False, True * ast.Num: int, float, complex * ast.Str: str * ast.Bytes: bytes * ast.Tuple if items are constants too: tuple * frozenset Update code to accept ast.Constant instead of ast.Num and/or ast.Str: * compiler * docstrings * ast.literal_eval() * Tools/parser/unparse.py --- Include/Python-ast.h | 13 ++++-- Include/asdl.h | 1 + Lib/ast.py | 52 ++++++++++++--------- Lib/test/test_ast.py | 120 +++++++++++++++++++++++++++++++++++++++++++++++- Misc/NEWS | 4 ++ Parser/Python.asdl | 7 ++- Parser/asdl.py | 3 +- Parser/asdl_c.py | 21 +++++++++ Python/Python-ast.c | 79 +++++++++++++++++++++++++++++++ Python/ast.c | 50 ++++++++++++++++++++ Python/compile.c | 60 ++++++++++++++++++------ Python/future.c | 5 +- Python/symtable.c | 1 + Tools/parser/unparse.py | 29 +++++++++++- 14 files changed, 401 insertions(+), 44 deletions(-) diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 175e380..1ab376a 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -202,9 +202,9 @@ enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, Await_kind=12, Yield_kind=13, YieldFrom_kind=14, Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18, FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21, - NameConstant_kind=22, Ellipsis_kind=23, Attribute_kind=24, - Subscript_kind=25, Starred_kind=26, Name_kind=27, - List_kind=28, Tuple_kind=29}; + NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24, + Attribute_kind=25, Subscript_kind=26, Starred_kind=27, + Name_kind=28, List_kind=29, Tuple_kind=30}; struct _expr { enum _expr_kind kind; union { @@ -316,6 +316,10 @@ struct _expr { } NameConstant; struct { + constant value; + } Constant; + + struct { expr_ty value; identifier attr; expr_context_ty ctx; @@ -567,6 +571,9 @@ expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); +#define Constant(a0, a1, a2, a3) _Py_Constant(a0, a1, a2, a3) +expr_ty _Py_Constant(constant value, int lineno, int col_offset, PyArena + *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); diff --git a/Include/asdl.h b/Include/asdl.h index 495153c..35e9fa1 100644 --- a/Include/asdl.h +++ b/Include/asdl.h @@ -6,6 +6,7 @@ typedef PyObject * string; typedef PyObject * bytes; typedef PyObject * object; typedef PyObject * singleton; +typedef PyObject * constant; /* It would be nice if the code generated by asdl_c.py was completely independent of Python, but it is a goal the requires too much work diff --git a/Lib/ast.py b/Lib/ast.py index 0170472..156a1f2 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -35,6 +35,8 @@ def parse(source, filename='', mode='exec'): return compile(source, filename, mode, PyCF_ONLY_AST) +_NUM_TYPES = (int, float, complex) + def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python @@ -47,7 +49,9 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): + if isinstance(node, Constant): + return node.value + elif isinstance(node, (Str, Bytes)): return node.s elif isinstance(node, Num): return node.n @@ -62,24 +66,21 @@ def literal_eval(node_or_string): in zip(node.keys, node.values)) elif isinstance(node, NameConstant): return node.value - elif isinstance(node, UnaryOp) and \ - isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): operand = _convert(node.operand) - if isinstance(node.op, UAdd): - return + operand - else: - return - operand - elif isinstance(node, BinOp) and \ - isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + if isinstance(operand, _NUM_TYPES): + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): left = _convert(node.left) right = _convert(node.right) - if isinstance(node.op, Add): - return left + right - else: - return left - right + if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES): + if isinstance(node.op, Add): + return left + right + else: + return left - right raise ValueError('malformed node or string: ' + repr(node)) return _convert(node_or_string) @@ -196,12 +197,19 @@ def get_docstring(node, clean=True): """ if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if not(node.body and isinstance(node.body[0], Expr)): + return + node = node.body[0].value + if isinstance(node, Str): + text = node.s + elif isinstance(node, Constant) and isinstance(node.value, str): + text = node.value + else: + return + if clean: + import inspect + text = inspect.cleandoc(text) + return text def walk(node): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d3e6d35..6d6c9bd 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,7 +1,8 @@ +import ast +import dis import os import sys import unittest -import ast import weakref from test import support @@ -933,6 +934,123 @@ class ASTValidatorTests(unittest.TestCase): compile(mod, fn, "exec") +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "", "exec") + + ns = {} + exec(code, ns) + return ns['x'] + + def test_singletons(self): + for const in (None, False, True, Ellipsis, b'', frozenset()): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = (123, 123.0, 123j, + "unicode", b'bytes', + tuple("tuple"), frozenset("frozenset"), + nested_tuple, nested_frozenset) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual(str(cm.exception), + "expression which can't be assigned " + "to in Store context") + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), 'docstring') + + tree.body[0].value = ast.Constant(value='constant docstring') + self.assertEqual(ast.get_docstring(tree), 'constant docstring') + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, '', 'exec') + consts = [] + for instr in dis.get_instructions(co): + if instr.opname == 'LOAD_CONST': + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, + True, False, + 124, + 2.0, + 3j, + "unicode", + b'bytes', + (1, 2, 3)] + + code = '\n'.join(map(repr, consts)) + code += '\n...' + + code_consts = [const for const in consts + if (not isinstance(const, (str, int, float, complex)) + or isinstance(const, bool))] + code_consts.append(Ellipsis) + # the compiler adds a final "LOAD_CONST None" + code_consts.append(None) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), code_consts) + + # Replace expression nodes with constants + for expr_node, const in zip(tree.body, consts): + assert isinstance(expr_node, ast.Expr) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, expr_node.value) + expr_node.value = new_node + + self.assertEqual(self.get_load_const(tree), code_consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 30) + + def main(): if __name__ != '__main__': return diff --git a/Misc/NEWS b/Misc/NEWS index 676f2ca..6e1a5a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26146: Add a new kind of AST node: ``ast.Constant``. It can be used + by external AST optimizers, but the compiler does not emit directly such + node. + - Issue #18018: Import raises ImportError instead of SystemError if a relative import is attempted without a known parent package. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 22775c6..aaddf61 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -1,4 +1,8 @@ --- ASDL's six builtin types are identifier, int, string, bytes, object, singleton +-- ASDL's 7 builtin types are: +-- identifier, int, string, bytes, object, singleton, constant +-- +-- singleton: None, True or False +-- constant can be None, whereas None means "no value" for object. module Python { @@ -76,6 +80,7 @@ module Python | Bytes(bytes s) | NameConstant(singleton value) | Ellipsis + | Constant(constant value) -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff --git a/Parser/asdl.py b/Parser/asdl.py index 121cdab..62f5c19 100644 --- a/Parser/asdl.py +++ b/Parser/asdl.py @@ -33,7 +33,8 @@ __all__ = [ # See the EBNF at the top of the file to understand the logical connection # between the various node types. -builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton'} +builtin_types = {'identifier', 'string', 'bytes', 'int', 'object', 'singleton', + 'constant'} class AST: def __repr__(self): diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index eedd89b..a81644d 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -834,6 +834,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -871,6 +872,26 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + /* don't increment the reference counter, Constant uses a borrowed + * reference, not a strong reference */ + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 07d9b3e..4dde11f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -306,6 +306,10 @@ static char *NameConstant_fields[]={ "value", }; static PyTypeObject *Ellipsis_type; +static PyTypeObject *Constant_type; +static char *Constant_fields[]={ + "value", +}; static PyTypeObject *Attribute_type; _Py_IDENTIFIER(attr); _Py_IDENTIFIER(ctx); @@ -709,6 +713,7 @@ static PyObject* ast2obj_object(void *o) return (PyObject*)o; } #define ast2obj_singleton ast2obj_object +#define ast2obj_constant ast2obj_object #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object #define ast2obj_bytes ast2obj_object @@ -746,6 +751,26 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) return 0; } +static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) +{ + if (obj == Py_None || obj == Py_True || obj == Py_False) { + /* don't increment the reference counter, Constant uses a borrowed + * reference, not a strong reference */ + *out = obj; + return 0; + } + + if (obj) { + if (PyArena_AddPyObject(arena, obj) < 0) { + *out = NULL; + return -1; + } + Py_INCREF(obj); + } + *out = obj; + return 0; +} + static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { @@ -941,6 +966,8 @@ static int init_types(void) if (!NameConstant_type) return 0; Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); if (!Ellipsis_type) return 0; + Constant_type = make_type("Constant", expr_type, Constant_fields, 1); + if (!Constant_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); @@ -2167,6 +2194,25 @@ Ellipsis(int lineno, int col_offset, PyArena *arena) } expr_ty +Constant(constant value, int lineno, int col_offset, PyArena *arena) +{ + expr_ty p; + if (!value) { + PyErr_SetString(PyExc_ValueError, + "field value is required for Constant"); + return NULL; + } + p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); + if (!p) + return NULL; + p->kind = Constant_kind; + p->v.Constant.value = value; + p->lineno = lineno; + p->col_offset = col_offset; + return p; +} + +expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena) { @@ -3267,6 +3313,15 @@ ast2obj_expr(void* _o) result = PyType_GenericNew(Ellipsis_type, NULL, NULL); if (!result) goto failed; break; + case Constant_kind: + result = PyType_GenericNew(Constant_type, NULL, NULL); + if (!result) goto failed; + value = ast2obj_constant(o->v.Constant.value); + if (!value) goto failed; + if (_PyObject_SetAttrId(result, &PyId_value, value) == -1) + goto failed; + Py_DECREF(value); + break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; @@ -6240,6 +6295,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (*out == NULL) goto failed; return 0; } + isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type); + if (isinstance == -1) { + return 1; + } + if (isinstance) { + constant value; + + if (_PyObject_HasAttrId(obj, &PyId_value)) { + int res; + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp == NULL) goto failed; + res = obj2ast_constant(tmp, &value, arena); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } else { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); + return 1; + } + *out = Constant(value, lineno, col_offset, arena); + if (*out == NULL) goto failed; + return 0; + } isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type); if (isinstance == -1) { return 1; @@ -7517,6 +7594,8 @@ PyInit__ast(void) 0) return NULL; if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) return NULL; + if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0) + return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0) diff --git a/Python/ast.c b/Python/ast.c index a5d8dba..5422e9c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -132,6 +132,50 @@ validate_arguments(arguments_ty args) } static int +validate_constant(PyObject *value) +{ + if (value == Py_None || value == Py_Ellipsis) + return 1; + + if (PyLong_CheckExact(value) + || PyFloat_CheckExact(value) + || PyComplex_CheckExact(value) + || PyBool_Check(value) + || PyUnicode_CheckExact(value) + || PyBytes_CheckExact(value)) + return 1; + + if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) { + PyObject *it; + + it = PyObject_GetIter(value); + if (it == NULL) + return 0; + + while (1) { + PyObject *item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(it); + return 0; + } + break; + } + + if (!validate_constant(item)) { + Py_DECREF(it); + return 0; + } + } + + Py_DECREF(it); + return 1; + } + + return 0; +} + +static int validate_expr(expr_ty exp, expr_context_ty ctx) { int check_ctx = 1; @@ -240,6 +284,12 @@ validate_expr(expr_ty exp, expr_context_ty ctx) return validate_expr(exp->v.Call.func, Load) && validate_exprs(exp->v.Call.args, Load, 0) && validate_keywords(exp->v.Call.keywords); + case Constant_kind: + if (!validate_constant(exp->v.Constant.value)) { + PyErr_SetString(PyExc_TypeError, "invalid type in Constant"); + return 0; + } + return 1; case Num_kind: { PyObject *n = exp->v.Num.n; if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) && diff --git a/Python/compile.c b/Python/compile.c index a710e82..ccb05cf 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1314,7 +1314,11 @@ compiler_isdocstring(stmt_ty s) { if (s->kind != Expr_kind) return 0; - return s->v.Expr.value->kind == Str_kind; + if (s->v.Expr.value->kind == Str_kind) + return 1; + if (s->v.Expr.value->kind == Constant_kind) + return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value); + return 0; } /* Compile a sequence of statements, checking for a docstring. */ @@ -1688,8 +1692,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) st = (stmt_ty)asdl_seq_GET(body, 0); docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; + if (docstring && c->c_optimize < 2) { + if (st->v.Expr.value->kind == Constant_kind) + first_const = st->v.Expr.value->v.Constant.value; + else + first_const = st->v.Expr.value->v.Str.s; + } if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { compiler_exit_scope(c); return 0; @@ -2600,6 +2608,36 @@ compiler_assert(struct compiler *c, stmt_ty s) } static int +compiler_visit_stmt_expr(struct compiler *c, expr_ty value) +{ + if (c->c_interactive && c->c_nestlevel <= 1) { + VISIT(c, expr, value); + ADDOP(c, PRINT_EXPR); + return 1; + } + + if (value->kind == Str_kind || value->kind == Num_kind) { + /* ignore strings and numbers */ + return 1; + } + + if (value->kind == Constant_kind) { + PyObject *cst = value->v.Constant.value; + if (PyUnicode_CheckExact(cst) + || PyLong_CheckExact(cst) + || PyFloat_CheckExact(cst) + || PyComplex_CheckExact(cst)) { + /* ignore strings and numbers */ + return 1; + } + } + + VISIT(c, expr, value); + ADDOP(c, POP_TOP); + return 1; +} + +static int compiler_visit_stmt(struct compiler *c, stmt_ty s) { Py_ssize_t i, n; @@ -2669,16 +2707,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s) case Nonlocal_kind: break; case Expr_kind: - if (c->c_interactive && c->c_nestlevel <= 1) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, PRINT_EXPR); - } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { - VISIT(c, expr, s->v.Expr.value); - ADDOP(c, POP_TOP); - } - break; + return compiler_visit_stmt_expr(c, s->v.Expr.value); case Pass_kind: break; case Break_kind: @@ -3625,6 +3654,8 @@ expr_constant(struct compiler *c, expr_ty e) switch (e->kind) { case Ellipsis_kind: return 1; + case Constant_kind: + return PyObject_IsTrue(e->v.Constant.value); case Num_kind: return PyObject_IsTrue(e->v.Num.n); case Str_kind: @@ -3912,6 +3943,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e) return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); + case Constant_kind: + ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts); + break; case Num_kind: ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); break; diff --git a/Python/future.c b/Python/future.c index 163f87f..75f2107 100644 --- a/Python/future.c +++ b/Python/future.c @@ -79,7 +79,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename) i = 0; first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i); - if (first->kind == Expr_kind && first->v.Expr.value->kind == Str_kind) + if (first->kind == Expr_kind + && (first->v.Expr.value->kind == Str_kind + || (first->v.Expr.value->kind == Constant_kind + && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value)))) i++; diff --git a/Python/symtable.c b/Python/symtable.c index 9f5149f..558bb06 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1455,6 +1455,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) case JoinedStr_kind: VISIT_SEQ(st, expr, e->v.JoinedStr.values); break; + case Constant_kind: case Num_kind: case Str_kind: case Bytes_kind: diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py index 9972797..35ebc66 100644 --- a/Tools/parser/unparse.py +++ b/Tools/parser/unparse.py @@ -343,6 +343,11 @@ class Unparser: value = t.s.replace("{", "{{").replace("}", "}}") write(value) + def _fstring_Constant(self, t, write): + assert isinstance(t.value, str) + value = t.value.replace("{", "{{").replace("}", "}}") + write(value) + def _fstring_FormattedValue(self, t, write): write("{") expr = io.StringIO() @@ -364,6 +369,25 @@ class Unparser: def _Name(self, t): self.write(t.id) + def _write_constant(self, value): + if isinstance(value, (float, complex)): + self.write(repr(value).replace("inf", INFSTR)) + else: + self.write(repr(value)) + + def _Constant(self, t): + value = t.value + if isinstance(value, tuple): + self.write("(") + if len(value) == 1: + self._write_constant(value[0]) + self.write(",") + else: + interleave(lambda: self.write(", "), self._write_constant, value) + self.write(")") + else: + self._write_constant(t.value) + def _NameConstant(self, t): self.write(repr(t.value)) @@ -443,7 +467,7 @@ class Unparser: def _Tuple(self, t): self.write("(") if len(t.elts) == 1: - (elt,) = t.elts + elt = t.elts[0] self.dispatch(elt) self.write(",") else: @@ -490,7 +514,8 @@ class Unparser: # Special case: 3.__abs__() is a syntax error, so if t.value # is an integer literal then we need to either parenthesize # it or add an extra space to get 3 .__abs__(). - if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + if ((isinstance(t.value, ast.Num) and isinstance(t.value.n, int)) + or (isinstance(t.value, ast.Constant) and isinstance(t.value.value, int))): self.write(" ") self.write(".") self.write(t.attr) -- cgit v0.12 From 906d82db6d63c81b9d63a0c99039358bce2da6cb Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 25 Jan 2016 23:00:21 -0800 Subject: Fix typo --- Lib/test/test_deque.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index c61e80b..f75b3ff 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -289,7 +289,7 @@ class TestBasic(unittest.TestCase): else: self.assertEqual(d.index(element, start, stop), target) - def test_insert_bug_24913(self): + def test_index_bug_24913(self): d = deque('A' * 3) with self.assertRaises(ValueError): i = d.index("Hello world", 0, 4) -- cgit v0.12 From 726f6902ce9c4db8dc73074ed5323a3bdeb04194 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:11:47 +0100 Subject: Fix a refleak in validate_constant() Issue #26146. --- Python/ast.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ast.c b/Python/ast.c index 5422e9c..ecfc14c 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -164,8 +164,10 @@ validate_constant(PyObject *value) if (!validate_constant(item)) { Py_DECREF(it); + Py_DECREF(item); return 0; } + Py_DECREF(item); } Py_DECREF(it); -- cgit v0.12 From 25219f596a069e8d4ed7114cd9b1bddc2a1de3b7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:37:59 +0100 Subject: Issue #26146: remove useless code obj2ast_constant() code is baesd on obj2ast_object() which has a special case for Py_None. But in practice, we don't need to have a special case for constants. Issue noticed by Joseph Jevnik on a review. --- Parser/asdl_c.py | 7 ------- Python/Python-ast.c | 7 ------- 2 files changed, 14 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index a81644d..17c8517 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -874,13 +874,6 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) { - if (obj == Py_None || obj == Py_True || obj == Py_False) { - /* don't increment the reference counter, Constant uses a borrowed - * reference, not a strong reference */ - *out = obj; - return 0; - } - if (obj) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 4dde11f..1193c7c 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -753,13 +753,6 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena) static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena) { - if (obj == Py_None || obj == Py_True || obj == Py_False) { - /* don't increment the reference counter, Constant uses a borrowed - * reference, not a strong reference */ - *out = obj; - return 0; - } - if (obj) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; -- cgit v0.12 From be59d1489bdf150e05e67a89cc772628af7e8fd6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 27 Jan 2016 00:39:12 +0100 Subject: Issue #26146: enhance ast.Constant error message Mention the name of the invalid type in error message of AST validation for constants. Suggestion made by Joseph Jevnik on a review. --- Lib/test/test_ast.py | 6 ++++++ Python/ast.c | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6d6c9bd..57060d8 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -951,6 +951,12 @@ class ConstantTests(unittest.TestCase): exec(code, ns) return ns['x'] + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), + "got an invalid type in Constant: list") + def test_singletons(self): for const in (None, False, True, Ellipsis, b'', frozenset()): with self.subTest(const=const): diff --git a/Python/ast.c b/Python/ast.c index ecfc14c..d19546a 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -288,7 +288,9 @@ validate_expr(expr_ty exp, expr_context_ty ctx) validate_keywords(exp->v.Call.keywords); case Constant_kind: if (!validate_constant(exp->v.Constant.value)) { - PyErr_SetString(PyExc_TypeError, "invalid type in Constant"); + PyErr_Format(PyExc_TypeError, + "got an invalid type in Constant: %s", + Py_TYPE(exp->v.Constant.value)->tp_name); return 0; } return 1; -- cgit v0.12 From 5586ba7643e608827fc7109a40e851f3cae0f2e4 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 30 Jan 2016 12:34:12 +0200 Subject: Simply docstrings of venv module This will hopefully make maintenance of venv documentation easier. For example, see commits a4f0d76af176 and 5764cc02244d. This patch has been reviewed by Vinaj Sajip, the maintainer of venv module. --- Lib/venv/__init__.py | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 74245ab..fa3d2a3 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -3,28 +3,6 @@ Virtual environment (venv) package for Python. Based on PEP 405. Copyright (C) 2011-2014 Vinay Sajip. Licensed to the PSF under a contributor agreement. - -usage: python -m venv [-h] [--system-site-packages] [--symlinks] [--clear] - [--upgrade] - ENV_DIR [ENV_DIR ...] - -Creates virtual Python environments in one or more target directories. - -positional arguments: - ENV_DIR A directory to create the environment in. - -optional arguments: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Attempt to symlink rather than copy. - --clear Delete the contents of the environment directory if it - already exists, before environment creation. - --upgrade Upgrade the environment directory to use this version - of Python, assuming Python has been upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) """ import logging import os @@ -349,23 +327,7 @@ class EnvBuilder: def create(env_dir, system_site_packages=False, clear=False, symlinks=False, with_pip=False): - """ - Create a virtual environment in a directory. - - By default, makes the system (global) site-packages dir *un*available to - the created environment, and uses copying rather than symlinking for files - obtained from the source Python installation. - - :param env_dir: The target directory to create an environment in. - :param system_site_packages: If True, the system (global) site-packages - dir is available to the environment. - :param clear: If True, delete the contents of the environment directory if - it already exists, before environment creation. - :param symlinks: If True, attempt to symlink rather than copy files into - virtual environment. - :param with_pip: If True, ensure pip is installed in the virtual - environment - """ + """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip) builder.create(env_dir) -- cgit v0.12 From ce5179fcbab40df1bf0a74a9ebf96cc7c1e061b3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 31 Jan 2016 08:56:21 -0800 Subject: Issue #23601: Use small object allocator for dict key objects --- Misc/NEWS | 3 +++ Objects/dictobject.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index aac1bad..a182887 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,9 @@ Core and Builtins by external AST optimizers, but the compiler does not emit directly such node. +- Issue #23601: Sped-up allocation of dict key objects by using Python's + small object allocator. (Contributed by Julian Taylor.) + - Issue #18018: Import raises ImportError instead of SystemError if a relative import is attempted without a known parent package. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f6a98b4..31a6322 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -324,7 +324,7 @@ static PyDictKeysObject *new_keys_object(Py_ssize_t size) assert(size >= PyDict_MINSIZE_SPLIT); assert(IS_POWER_OF_2(size)); - dk = PyMem_MALLOC(sizeof(PyDictKeysObject) + + dk = PyObject_MALLOC(sizeof(PyDictKeysObject) + sizeof(PyDictKeyEntry) * (size-1)); if (dk == NULL) { PyErr_NoMemory(); @@ -353,7 +353,7 @@ free_keys_object(PyDictKeysObject *keys) Py_XDECREF(entries[i].me_key); Py_XDECREF(entries[i].me_value); } - PyMem_FREE(keys); + PyObject_FREE(keys); } #define new_values(size) PyMem_NEW(PyObject *, size) @@ -964,7 +964,7 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) } } assert(oldkeys->dk_refcnt == 1); - DK_DEBUG_DECREF PyMem_FREE(oldkeys); + DK_DEBUG_DECREF PyObject_FREE(oldkeys); } return 0; } -- cgit v0.12 From a63897164eb1fb01bdc51fb8de0d6bc7eb791de9 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 1 Feb 2016 21:21:19 -0800 Subject: merge --- Doc/library/collections.rst | 4 ++-- Lib/test/test_deque.py | 23 ++++++++++++----------- Modules/_collectionsmodule.c | 7 ++----- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 6a0c127..080f968 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -477,8 +477,8 @@ or subtracting from an empty counter. Insert *x* into the deque at position *i*. - If the insertion causes a bounded deque to grow beyond *maxlen*, the - rightmost element is then removed to restore the size limit. + If the insertion would cause a bounded deque to grow beyond *maxlen*, + an :exc:`IndexError` is raised. .. versionadded:: 3.5 diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 34b1be2..e2c6b6c 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -304,19 +304,20 @@ class TestBasic(unittest.TestCase): s.insert(i, 'Z') self.assertEqual(list(d), s) - def test_index_bug_26194(self): + def test_insert_bug_26194(self): data = 'ABC' - for i in range(len(data) + 1): - d = deque(data, len(data)) - d.insert(i, None) - s = list(data) - s.insert(i, None) - s.pop() - self.assertEqual(list(d), s) - if i < len(data): - self.assertIsNone(d[i]) + d = deque(data, maxlen=len(data)) + with self.assertRaises(IndexError): + d.insert(2, None) + + elements = 'ABCDEFGHI' + for i in range(-len(elements), len(elements)): + d = deque(elements, maxlen=len(elements)+1) + d.insert(i, 'Z') + if i >= 0: + self.assertEqual(d[i], 'Z') else: - self.assertTrue(None not in d) + self.assertEqual(d[i-1], 'Z') def test_imul(self): for n in (-10, -1, 0, 1, 2, 10, 1000): diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 69acc64..479b052 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1085,16 +1085,13 @@ deque_insert(dequeobject *deque, PyObject *args) Py_ssize_t index; Py_ssize_t n = Py_SIZE(deque); PyObject *value; - PyObject *oldvalue; PyObject *rv; if (!PyArg_ParseTuple(args, "nO:insert", &index, &value)) return NULL; if (deque->maxlen == Py_SIZE(deque)) { - if (index >= deque->maxlen || Py_SIZE(deque) == 0) - Py_RETURN_NONE; - oldvalue = deque_pop(deque, NULL); - Py_DECREF(oldvalue); + PyErr_SetString(PyExc_IndexError, "deque already at its maximum size"); + return NULL; } if (index >= n) return deque_append(deque, value); -- cgit v0.12 From c9deece272abe87865d91f3077fe4591e57303ad Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 3 Feb 2016 05:19:44 +0000 Subject: Issue #24421: Compile _math.c separately to avoid race condition --- Makefile.pre.in | 6 +++++- Misc/NEWS | 4 ++++ setup.py | 12 ++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 57167e6..2a687e5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -586,11 +586,15 @@ pybuilddir.txt: $(BUILDPYTHON) exit 1 ; \ fi +# This is shared by the math and cmath modules +Modules/_math.o: Modules/_math.c Modules/_math.h + $(CC) -c $(CCSHARED) $(PY_CORE_CFLAGS) -o $@ $< + # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) pybuilddir.txt +sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ diff --git a/Misc/NEWS b/Misc/NEWS index a182887..4532488 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -671,6 +671,10 @@ Build - Issue #24986: It is now possible to build Python on Windows without errors when external libraries are not available. +- Issue #24421: Compile Modules/_math.c once, before building extensions. + Previously it could fail to compile properly if the math and cmath builds + were concurrent. + - Issue #25798: Update OS X 10.5 installer to use OpenSSL 1.0.2e. Windows diff --git a/setup.py b/setup.py index da67731..930b64a 100644 --- a/setup.py +++ b/setup.py @@ -582,13 +582,17 @@ class PyBuildExt(build_ext): # array objects exts.append( Extension('array', ['arraymodule.c']) ) + + shared_math = 'Modules/_math.o' # complex math library functions - exts.append( Extension('cmath', ['cmathmodule.c', '_math.c'], - depends=['_math.h'], + exts.append( Extension('cmath', ['cmathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], libraries=math_libs) ) # math library functions, e.g. sin() - exts.append( Extension('math', ['mathmodule.c', '_math.c'], - depends=['_math.h'], + exts.append( Extension('math', ['mathmodule.c'], + extra_objects=[shared_math], + depends=['_math.h', shared_math], libraries=math_libs) ) # time libraries: librt may be needed for clock_gettime() -- cgit v0.12 From f50215412cdc41b7845a6c0bb040a8f3521a0b84 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 4 Feb 2016 02:46:16 -0800 Subject: Add early-out for the common case where kwds is NULL (gives 1.1% speedup). --- Objects/setobject.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 8cb3f36..c9834a8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1088,7 +1088,8 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { PyObject *iterable = NULL, *result; - if (type == &PyFrozenSet_Type && !_PyArg_NoKeywords("frozenset()", kwds)) + if (kwds != NULL && type == &PyFrozenSet_Type + && !_PyArg_NoKeywords("frozenset()", kwds)) return NULL; if (!PyArg_UnpackTuple(args, type->tp_name, 0, 1, &iterable)) @@ -1130,7 +1131,7 @@ PySet_Fini(void) static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) + if (kwds != NULL && type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) return NULL; return make_new_set(type, NULL); -- cgit v0.12 From 135d5f49f6da605bb1073f939e826e352a2d655f Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 5 Feb 2016 18:23:08 -0500 Subject: Fix issue 26287: While handling FORMAT_VALUE opcode, the top of stack was being corrupted if an error occurred in PyObject_Format(). --- Lib/test/test_fstring.py | 11 +++++++++++ Python/ceval.c | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index d6f781c..a82dedf 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -692,6 +692,17 @@ f'{a * x()}'""" r"f'{a(4]}'", ]) + def test_errors(self): + # see issue 26287 + self.assertAllRaise(TypeError, 'non-empty', + [r"f'{(lambda: 0):x}'", + r"f'{(0,):x}'", + ]) + self.assertAllRaise(ValueError, 'Unknown format code', + [r"f'{1000:j}'", + r"f'{1000:j}'", + ]) + def test_loop(self): for i in range(1000): self.assertEqual(f'i:{i}', 'i:' + str(i)) diff --git a/Python/ceval.c b/Python/ceval.c index 743a969..b815ccd 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3383,7 +3383,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) int have_fmt_spec = (oparg & FVS_MASK) == FVS_HAVE_SPEC; fmt_spec = have_fmt_spec ? POP() : NULL; - value = TOP(); + value = POP(); /* See if any conversion is specified. */ switch (which_conversion) { @@ -3426,7 +3426,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) goto error; } - SET_TOP(result); + PUSH(result); DISPATCH(); } -- cgit v0.12 From eb588a1d10e94085c622eb732cfe535e0ebaec8a Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Fri, 5 Feb 2016 18:26:20 -0500 Subject: Switch to more idiomatic C code. --- Python/ceval.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b815ccd..8904d7a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3399,10 +3399,10 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) /* If there's a conversion function, call it and replace value with that result. Otherwise, just use value, without conversion. */ - if (conv_fn) { + if (conv_fn != NULL) { result = conv_fn(value); Py_DECREF(value); - if (!result) { + if (result == NULL) { Py_XDECREF(fmt_spec); goto error; } @@ -3422,8 +3422,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) result = PyObject_Format(value, fmt_spec); Py_DECREF(value); Py_XDECREF(fmt_spec); - if (!result) + if (result == NULL) { goto error; + } } PUSH(result); -- cgit v0.12 From 186c30b7ae2c304b863143809867cfef7eb1bf29 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 5 Feb 2016 19:40:01 -0500 Subject: Issue #26288: Optimize PyLong_AsDouble. --- Misc/NEWS | 1 + Objects/longobject.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index dcccc4a..510748b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -165,6 +165,7 @@ Core and Builtins - Issue #25660: Fix TAB key behaviour in REPL with readline. +- Issue #26288: Optimize PyLong_AsDouble. Library ------- diff --git a/Objects/longobject.c b/Objects/longobject.c index d05de8b..c1edeac 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2769,6 +2769,13 @@ PyLong_AsDouble(PyObject *v) PyErr_SetString(PyExc_TypeError, "an integer is required"); return -1.0; } + if (Py_ABS(Py_SIZE(v)) <= 1) { + /* Fast path; single digit will always fit decimal. + This improves performance of FP/long operations by at + least 20%. This is even visible on macro-benchmarks. + */ + return (double)MEDIUM_VALUE((PyLongObject *)v); + } x = _PyLong_Frexp((PyLongObject *)v, &exponent); if ((x == -1.0 && PyErr_Occurred()) || exponent > DBL_MAX_EXP) { PyErr_SetString(PyExc_OverflowError, -- cgit v0.12 From a0fcaca4e1c29078f578fb96971398c70cf38900 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 6 Feb 2016 12:21:33 -0500 Subject: Issue #26288: Fix comment --- Objects/longobject.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index c1edeac..f3afb10 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2770,9 +2770,9 @@ PyLong_AsDouble(PyObject *v) return -1.0; } if (Py_ABS(Py_SIZE(v)) <= 1) { - /* Fast path; single digit will always fit decimal. - This improves performance of FP/long operations by at - least 20%. This is even visible on macro-benchmarks. + /* Fast path; single digit long (31 bits) will cast safely + to double. This improves performance of FP/long operations + by 20%. */ return (double)MEDIUM_VALUE((PyLongObject *)v); } -- cgit v0.12 From 503f908090bd22df70ed21e3bb3c2513a1eee969 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 00:02:25 +0200 Subject: Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). Patch by Thomas Kluyver. --- Doc/library/zipfile.rst | 24 ++++++++++++++++ Doc/whatsnew/3.6.rst | 10 +++++++ Lib/test/test_zipfile.py | 15 ++++++++++ Lib/zipfile.py | 75 ++++++++++++++++++++++++++++++------------------ Misc/NEWS | 3 ++ 5 files changed, 99 insertions(+), 28 deletions(-) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index e3c2217..54b8cc5 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -465,6 +465,22 @@ Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and :meth:`.infolist` methods of :class:`ZipFile` objects. Each object stores information about a single member of the ZIP archive. +There is one classmethod to make a :class:`ZipInfo` instance for a filesystem +file: + +.. classmethod:: ZipInfo.from_file(filename, arcname=None) + + Construct a :class:`ZipInfo` instance for a file on the filesystem, in + preparation for adding it to a zip file. + + *filename* should be the path to a file or directory on the filesystem. + + If *arcname* is specified, it is used as the name within the archive. + If *arcname* is not specified, the name will be the same as *filename*, but + with any drive letter and leading path separators removed. + + .. versionadded:: 3.6 + Instances have the following attributes: @@ -574,3 +590,11 @@ Instances have the following attributes: .. attribute:: ZipInfo.file_size Size of the uncompressed file. + +There is one method: + +.. method:: ZipInfo.is_dir() + + Return ``True`` if the ZipInfo represents a directory. + + .. versionadded:: 3.6 diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 158dad3..b7cc159 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -140,6 +140,16 @@ urllib.robotparser (Contributed by Nikolay Bogoychev in :issue:`16099`.) +zipfile +------- + +A new :meth:`ZipInfo.from_file() ` class method +allow to make :class:`~zipfile.ZipInfo` instance from a filesystem file. +A new :meth:`ZipInfo.is_dir() ` method can be used +to check if the :class:`~zipfile.ZipInfo` instance represents a directory. +(Contributed by Thomas Kluyver in :issue:`26039`.) + + Optimizations ============= diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 2c10821..8589342 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -3,6 +3,7 @@ import io import os import sys import importlib.util +import posixpath import time import struct import zipfile @@ -2071,5 +2072,19 @@ class LzmaUniversalNewlineTests(AbstractUniversalNewlineTests, unittest.TestCase): compression = zipfile.ZIP_LZMA +class ZipInfoTests(unittest.TestCase): + def test_from_file(self): + zi = zipfile.ZipInfo.from_file(__file__) + self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertFalse(zi.is_dir()) + + def test_from_dir(self): + dirpath = os.path.dirname(os.path.abspath(__file__)) + zi = zipfile.ZipInfo.from_file(dirpath, 'stdlib_tests') + self.assertEqual(zi.filename, 'stdlib_tests/') + self.assertTrue(zi.is_dir()) + self.assertEqual(zi.compress_type, zipfile.ZIP_STORED) + self.assertEqual(zi.file_size, 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 56a2479..e0598d2 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -371,7 +371,7 @@ class ZipInfo (object): result.append(' filemode=%r' % stat.filemode(hi)) if lo: result.append(' external_attr=%#x' % lo) - isdir = self.filename[-1:] == '/' + isdir = self.is_dir() if not isdir or self.file_size: result.append(' file_size=%r' % self.file_size) if ((not isdir or self.compress_size) and @@ -469,6 +469,41 @@ class ZipInfo (object): extra = extra[ln+4:] + @classmethod + def from_file(cls, filename, arcname=None): + """Construct an appropriate ZipInfo for a file on the filesystem. + + filename should be the path to a file or directory on the filesystem. + + arcname is the name which it will have within the archive (by default, + this will be the same as filename, but without a drive letter and with + leading path separators removed). + """ + st = os.stat(filename) + isdir = stat.S_ISDIR(st.st_mode) + mtime = time.localtime(st.st_mtime) + date_time = mtime[0:6] + # Create ZipInfo instance to store file information + if arcname is None: + arcname = filename + arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) + while arcname[0] in (os.sep, os.altsep): + arcname = arcname[1:] + if isdir: + arcname += '/' + zinfo = cls(arcname, date_time) + zinfo.external_attr = (st.st_mode & 0xFFFF) << 16 # Unix attributes + if isdir: + zinfo.file_size = 0 + zinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + zinfo.file_size = st.st_size + + return zinfo + + def is_dir(self): + return self.filename[-1] == '/' + class _ZipDecrypter: """Class to handle decryption of files stored within a ZIP archive. @@ -1389,7 +1424,7 @@ class ZipFile: if upperdirs and not os.path.exists(upperdirs): os.makedirs(upperdirs) - if member.filename[-1] == '/': + if member.is_dir(): if not os.path.isdir(targetpath): os.mkdir(targetpath) return targetpath @@ -1430,29 +1465,17 @@ class ZipFile: raise RuntimeError( "Attempt to write to ZIP archive that was already closed") - st = os.stat(filename) - isdir = stat.S_ISDIR(st.st_mode) - mtime = time.localtime(st.st_mtime) - date_time = mtime[0:6] - # Create ZipInfo instance to store file information - if arcname is None: - arcname = filename - arcname = os.path.normpath(os.path.splitdrive(arcname)[1]) - while arcname[0] in (os.sep, os.altsep): - arcname = arcname[1:] - if isdir: - arcname += '/' - zinfo = ZipInfo(arcname, date_time) - zinfo.external_attr = (st[0] & 0xFFFF) << 16 # Unix attributes - if isdir: - zinfo.compress_type = ZIP_STORED - elif compress_type is None: - zinfo.compress_type = self.compression + zinfo = ZipInfo.from_file(filename, arcname) + + if zinfo.is_dir(): + zinfo.compress_size = 0 + zinfo.CRC = 0 else: - zinfo.compress_type = compress_type + if compress_type is not None: + zinfo.compress_type = compress_type + else: + zinfo.compress_type = self.compression - zinfo.file_size = st.st_size - zinfo.flag_bits = 0x00 with self._lock: if self._seekable: self.fp.seek(self.start_dir) @@ -1464,11 +1487,7 @@ class ZipFile: self._writecheck(zinfo) self._didModify = True - if isdir: - zinfo.file_size = 0 - zinfo.compress_size = 0 - zinfo.CRC = 0 - zinfo.external_attr |= 0x10 # MS-DOS directory flag + if zinfo.is_dir(): self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo self.fp.write(zinfo.FileHeader(False)) diff --git a/Misc/NEWS b/Misc/NEWS index 510748b..a7c3f58 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Core and Builtins Library ------- +- Issue #26039: Added zipfile.ZipInfo.from_file() and zipinfo.ZipInfo.is_dir(). + Patch by Thomas Kluyver. + - Issue #12923: Reset FancyURLopener's redirect counter even if there is an exception. Based on patches by Brian Brazil and Daniel Rocco. -- cgit v0.12 From 4cd63ef67a3e0974f0c48c550769babd401075e3 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 01:22:47 +0200 Subject: Issue #26198: ValueError is now raised instead of TypeError on buffer overflow in parsing "es#" and "et#" format units. SystemError is now raised instead of TypeError on programmical error in parsing format string. --- Lib/test/test_capi.py | 4 ++-- Lib/test/test_getargs2.py | 8 ++++---- Misc/NEWS | 7 +++++++ Python/getargs.c | 9 +++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 96666d1..1a743fd 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -488,10 +488,10 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(tuple_1, dict_b, format.encode("ascii"), keywords) when_not_skipped = False - except TypeError as e: + except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except RuntimeError as e: + except (TypeError, RuntimeError): when_not_skipped = False # test the format unit when skipped diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 6491135..be777af 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -636,10 +636,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_es_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_es_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_es_hash, 'abc\xe9', 'latin1', buf) def test_et_hash(self): from _testcapi import getargs_et_hash @@ -662,10 +662,10 @@ class String_TestCase(unittest.TestCase): self.assertEqual(getargs_et_hash('abc\xe9', 'latin1', buf), b'abc\xe9') self.assertEqual(buf, bytearray(b'abc\xe9\x00')) buf = bytearray(b'x'*4) - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) self.assertEqual(buf, bytearray(b'x'*4)) buf = bytearray() - self.assertRaises(TypeError, getargs_et_hash, 'abc\xe9', 'latin1', buf) + self.assertRaises(ValueError, getargs_et_hash, 'abc\xe9', 'latin1', buf) def test_u(self): from _testcapi import getargs_u diff --git a/Misc/NEWS b/Misc/NEWS index a7c3f58..7a198d6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -713,6 +713,13 @@ Tools/Demos - Issue #25154: The pyvenv script has been deprecated in favour of `python3 -m venv`. +C API +----- + +- Issue #26198: ValueError is now raised instead of TypeError on buffer + overflow in parsing "es#" and "et#" format units. SystemError is now raised + instead of TypeError on programmical error in parsing format string. + What's New in Python 3.5.1 final? ================================= diff --git a/Python/getargs.c b/Python/getargs.c index 60b7677..be6e375 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -394,7 +394,12 @@ seterror(Py_ssize_t iarg, const char *msg, int *levels, const char *fname, PyOS_snprintf(p, sizeof(buf) - (p - buf), " %.256s", msg); message = buf; } - PyErr_SetString(PyExc_TypeError, message); + if (msg[0] == '(') { + PyErr_SetString(PyExc_SystemError, message); + } + else { + PyErr_SetString(PyExc_TypeError, message); + } } @@ -1129,7 +1134,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { if (size + 1 > BUFFER_LEN) { Py_DECREF(s); - PyErr_Format(PyExc_TypeError, + PyErr_Format(PyExc_ValueError, "encoded string too long " "(%zd, maximum length %zd)", (Py_ssize_t)size, (Py_ssize_t)(BUFFER_LEN-1)); -- cgit v0.12 From d2962f145a4ad73ac5dad8ff9b3b26905d44d682 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 8 Feb 2016 16:39:05 +0200 Subject: Issue #25949: __dict__ for an OrderedDict instance is now created only when needed. --- Lib/test/test_ordered_dict.py | 4 +++- Misc/NEWS | 3 +++ Objects/odictobject.c | 23 +++++++---------------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 633e909..e1564ba 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -298,9 +298,11 @@ class OrderedDictTests: # do not save instance dictionary if not needed pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) + self.assertIsInstance(od.__dict__, dict) self.assertIsNone(od.__reduce__()[2]) od.x = 10 - self.assertIsNotNone(od.__reduce__()[2]) + self.assertEqual(od.__dict__['x'], 10) + self.assertEqual(od.__reduce__()[2], {'x': 10}) def test_pickle_recursive(self): OrderedDict = self.OrderedDict diff --git a/Misc/NEWS b/Misc/NEWS index bb7c57d..e6590b5 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -170,6 +170,9 @@ Core and Builtins Library ------- +- Issue #25949: __dict__ for an OrderedDict instance is now created only when + needed. + - Issue #25911: Restored support of bytes paths in os.walk() on Windows. - Issue #26045: Add UTF-8 suggestion to error message when posting a diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 1abdd02..dccbb3e 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1424,14 +1424,13 @@ static PyMethodDef odict_methods[] = { * OrderedDict members */ -/* tp_members */ +/* tp_getset */ -static PyMemberDef odict_members[] = { - {"__dict__", T_OBJECT, offsetof(PyODictObject, od_inst_dict), READONLY}, - {0} +static PyGetSetDef odict_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}, + {NULL} }; - /* ---------------------------------------------- * OrderedDict type slot methods */ @@ -1653,20 +1652,12 @@ odict_init(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * odict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *dict; PyODictObject *od; - dict = PyDict_New(); - if (dict == NULL) - return NULL; - od = (PyODictObject *)PyDict_Type.tp_new(type, args, kwds); - if (od == NULL) { - Py_DECREF(dict); + if (od == NULL) return NULL; - } - od->od_inst_dict = dict; /* type constructor fills the memory with zeros (see PyType_GenericAlloc()), there is no need to set them to zero again */ if (_odict_resize(od) < 0) { @@ -1708,8 +1699,8 @@ PyTypeObject PyODict_Type = { (getiterfunc)odict_iter, /* tp_iter */ 0, /* tp_iternext */ odict_methods, /* tp_methods */ - odict_members, /* tp_members */ - 0, /* tp_getset */ + 0, /* tp_members */ + odict_getset, /* tp_getset */ &PyDict_Type, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ -- cgit v0.12 From f089196beb0ad8fb262be42eddd92af3f0fe81c1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:15:21 +0100 Subject: Simplify main() of test_ast * Use ast.parse() to get the AST for a statement * Use str%args syntax for format a line Issue #26204. --- Lib/test/test_ast.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 57060d8..f5c4ade 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1064,8 +1064,9 @@ def main(): for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), (eval_tests, "eval")): print(kind+"_results = [") - for s in statements: - print(repr(to_tuple(compile(s, "?", kind, 0x400)))+",") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) print("]") print("main()") raise SystemExit -- cgit v0.12 From 51d8c526d5634fa1f0e4976fd357c5423a792082 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 17:57:02 +0100 Subject: Replace noop constant statement with expression * Constant statements will be ignored and the compiler will emit a SyntaxWarning. * Replace constant statement (ex: "1") with an expression statement (ex: "x=1"). * test_traceback: use context manager on the file. Issue #26204. --- Lib/test/test_inspect.py | 2 +- Lib/test/test_peepholer.py | 20 +++++++++++--------- Lib/test/test_support.py | 3 ++- Lib/test/test_sys_settrace.py | 4 ++-- Lib/test/test_traceback.py | 14 +++++++------- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 422a3d6..98d596e 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -401,7 +401,7 @@ class TestRetrievingSourceCode(GetSourceBase): self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) fn = "_non_existing_filename_used_for_sourcefile_test.py" - co = compile("None", fn, "exec") + co = compile("x=1", fn, "exec") self.assertEqual(inspect.getsourcefile(co), None) linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) try: diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 41e5091..b033640 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,9 +1,8 @@ import dis import re import sys -from io import StringIO +import textwrap import unittest -from math import copysign from test.bytecode_helper import BytecodeTestCase @@ -30,22 +29,25 @@ class TestTranforms(BytecodeTestCase): def test_global_as_constant(self): # LOAD_GLOBAL None/True/False --> LOAD_CONST None/True/False - def f(x): - None - None + def f(): + x = None + x = None return x - def g(x): - True + def g(): + x = True return x - def h(x): - False + def h(): + x = False return x + for func, elem in ((f, None), (g, True), (h, False)): self.assertNotInBytecode(func, 'LOAD_GLOBAL') self.assertInBytecode(func, 'LOAD_CONST', elem) + def f(): 'Adding a docstring made this test fail in Py2.5.0' return None + self.assertNotInBytecode(f, 'LOAD_GLOBAL') self.assertInBytecode(f, 'LOAD_CONST', None) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index f86ea91..269d9bf 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -230,7 +230,8 @@ class TestSupport(unittest.TestCase): def test_check_syntax_error(self): support.check_syntax_error(self, "def class") - self.assertRaises(AssertionError, support.check_syntax_error, self, "1") + with self.assertRaises(AssertionError): + support.check_syntax_error(self, "x=1") def test_CleanImport(self): import importlib diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index ae8f845..bb83623 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -338,8 +338,8 @@ class TraceTestCase(unittest.TestCase): def test_14_onliner_if(self): def onliners(): - if True: False - else: True + if True: x=False + else: x=True return 0 self.run_and_compare( onliners, diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b7695d6..2f739ba 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -129,12 +129,12 @@ class SyntaxTracebackCases(unittest.TestCase): def do_test(firstlines, message, charset, lineno): # Raise the message in a subprocess, and catch the output try: - output = open(TESTFN, "w", encoding=charset) - output.write("""{0}if 1: - import traceback; - raise RuntimeError('{1}') - """.format(firstlines, message)) - output.close() + with open(TESTFN, "w", encoding=charset) as output: + output.write("""{0}if 1: + import traceback; + raise RuntimeError('{1}') + """.format(firstlines, message)) + process = subprocess.Popen([sys.executable, TESTFN], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = process.communicate() @@ -176,7 +176,7 @@ class SyntaxTracebackCases(unittest.TestCase): do_test(" \t\f\n# coding: {0}\n".format(charset), text, charset, 5) # Issue #18960: coding spec should has no effect - do_test("0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) + do_test("x=0\n# coding: GBK\n", "h\xe9 ho", 'utf-8', 5) def test_print_traceback_at_exit(self): # Issue #22599: Ensure that it is possible to use the traceback module -- cgit v0.12 From a2724095cd55d75497d9cd48d35911599f4dedda Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 18:17:58 +0100 Subject: compiler now ignores constant statements The compile ignores constant statements and emit a SyntaxWarning warning. Don't emit the warning for string statement because triple quoted string is a common syntax for multiline comments. Don't emit the warning on ellipis neither: 'def f(): ...' is a legit syntax for abstract functions. Changes: * test_ast: ignore SyntaxWarning when compiling test statements. Modify test_load_const() to use assignment expressions rather than constant expression. * test_code: add more kinds of constant statements, ignore SyntaxWarning when testing that the compiler removes constant statements. * test_grammar: ignore SyntaxWarning on the statement "1" --- Lib/test/test_ast.py | 33 +++++++++++++-------------- Lib/test/test_code.py | 58 +++++++++++++++++++++++++++++++++--------------- Lib/test/test_grammar.py | 6 ++++- Misc/NEWS | 4 ++++ Python/compile.c | 41 +++++++++++++++++++++++++--------- 5 files changed, 95 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index f5c4ade..dbcd9f7 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,6 +3,7 @@ import dis import os import sys import unittest +import warnings import weakref from test import support @@ -239,8 +240,10 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i): - compile(ast_tree, "?", kind) + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice @@ -1020,27 +1023,23 @@ class ConstantTests(unittest.TestCase): b'bytes', (1, 2, 3)] - code = '\n'.join(map(repr, consts)) - code += '\n...' - - code_consts = [const for const in consts - if (not isinstance(const, (str, int, float, complex)) - or isinstance(const, bool))] - code_consts.append(Ellipsis) - # the compiler adds a final "LOAD_CONST None" - code_consts.append(None) + code = '\n'.join(['x={!r}'.format(const) for const in consts]) + code += '\nx = ...' + consts.extend((Ellipsis, None)) tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) # Replace expression nodes with constants - for expr_node, const in zip(tree.body, consts): - assert isinstance(expr_node, ast.Expr) + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) new_node = ast.Constant(value=const) - ast.copy_location(new_node, expr_node.value) - expr_node.value = new_node + ast.copy_location(new_node, assign.value) + assign.value = new_node - self.assertEqual(self.get_load_const(tree), code_consts) + self.assertEqual(self.get_load_const(tree), + consts) def test_literal_eval(self): tree = ast.parse("1 + 2") diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 21b12a5..2cf088c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,24 +66,6 @@ nlocals: 1 flags: 67 consts: ('None',) ->>> def optimize_away(): -... 'doc string' -... 'not a docstring' -... 53 -... 0x53 - ->>> dump(optimize_away.__code__) -name: optimize_away -argcount: 0 -kwonlyargcount: 0 -names: () -varnames: () -cellvars: () -freevars: () -nlocals: 0 -flags: 67 -consts: ("'doc string'", 'None') - >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -102,8 +84,10 @@ consts: ('None',) """ +import textwrap import unittest import weakref +import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -134,6 +118,44 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + def dump(self, co): + dump = {} + for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", + "cellvars", "freevars", "nlocals", "flags"]: + dump[attr] = getattr(co, "co_" + attr) + dump['consts'] = tuple(consts(co.co_consts)) + return dump + + def test_optimize_away(self): + ns = {} + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=SyntaxWarning) + exec(textwrap.dedent(''' + def optimize_away(): + 'doc string' + 'not a docstring' + 53 + 0x53 + b'bytes' + 1.0 + True + False + None + ... + '''), ns) + + self.assertEqual(self.dump(ns['optimize_away'].__code__), + {'name': 'optimize_away', + 'argcount': 0, + 'kwonlyargcount': 0, + 'names': (), + 'varnames': (), + 'cellvars': (), + 'freevars': (), + 'nlocals': 0, + 'flags': 67, + 'consts': ("'doc string'", 'None')}) + class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 8f8d71c..2d6f5ed 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,6 +7,7 @@ import unittest import sys # testing import * from sys import * +from test import support class TokenTests(unittest.TestCase): @@ -424,8 +425,11 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): + msg = 'ignore constant statement' + with support.check_warnings((msg, SyntaxWarning)): + exec("1") + # (exprlist '=')* exprlist - 1 1, 2, 3 x = 1 x = 1, 2, 3 diff --git a/Misc/NEWS b/Misc/NEWS index 0c7d18e..e1875c0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26204: The compiler now ignores constant statements (ex: "def f(): 1") + and emit a SyntaxWarning warning. The warning is not emitted for string and + ellipsis (...) statements. + - Issue #4806: Avoid masking the original TypeError exception when using star (*) unpacking in function calls. Based on patch by Hagen Fürstenau and Daniel Urban. diff --git a/Python/compile.c b/Python/compile.c index ccb05cf..84b79a2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2616,20 +2616,39 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value) return 1; } - if (value->kind == Str_kind || value->kind == Num_kind) { - /* ignore strings and numbers */ + switch (value->kind) + { + case Str_kind: + case Ellipsis_kind: + /* Issue #26204: ignore string statement, but don't emit a + * SyntaxWarning. Triple quoted strings is a common syntax for + * multiline comments. + * + * Don't emit warning on "def f(): ..." neither. It's a legit syntax + * for abstract function. */ return 1; - } - if (value->kind == Constant_kind) { - PyObject *cst = value->v.Constant.value; - if (PyUnicode_CheckExact(cst) - || PyLong_CheckExact(cst) - || PyFloat_CheckExact(cst) - || PyComplex_CheckExact(cst)) { - /* ignore strings and numbers */ - return 1; + case Bytes_kind: + case Num_kind: + case NameConstant_kind: + case Constant_kind: + { + PyObject *msg = PyUnicode_FromString("ignore constant statement"); + if (msg == NULL) + return 0; + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, + msg, + c->c_filename, c->u->u_lineno, + NULL, NULL) == -1) { + Py_DECREF(msg); + return 0; } + Py_DECREF(msg); + return 1; + } + + default: + break; } VISIT(c, expr, value); -- cgit v0.12 From 15a3095d64e96d0fe7448270f2c5b0bf22f9c4e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Feb 2016 22:45:06 +0100 Subject: compiler: don't emit SyntaxWarning on const stmt Issue #26204: the compiler doesn't emit SyntaxWarning warnings anymore when constant statements are ignored. --- Lib/test/test_ast.py | 7 ++---- Lib/test/test_code.py | 58 +++++++++++++++--------------------------------- Lib/test/test_grammar.py | 6 +---- Misc/NEWS | 7 +++--- Python/compile.c | 24 ++------------------ 5 files changed, 27 insertions(+), 75 deletions(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index dbcd9f7..a025c20 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3,7 +3,6 @@ import dis import os import sys import unittest -import warnings import weakref from test import support @@ -240,10 +239,8 @@ class AST_Tests(unittest.TestCase): ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) self.assertEqual(to_tuple(ast_tree), o) self._assertTrueorder(ast_tree, (0, 0)) - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 2cf088c..21b12a5 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -66,6 +66,24 @@ nlocals: 1 flags: 67 consts: ('None',) +>>> def optimize_away(): +... 'doc string' +... 'not a docstring' +... 53 +... 0x53 + +>>> dump(optimize_away.__code__) +name: optimize_away +argcount: 0 +kwonlyargcount: 0 +names: () +varnames: () +cellvars: () +freevars: () +nlocals: 0 +flags: 67 +consts: ("'doc string'", 'None') + >>> def keywordonly_args(a,b,*,k1): ... return a,b,k1 ... @@ -84,10 +102,8 @@ consts: ('None',) """ -import textwrap import unittest import weakref -import warnings from test.support import run_doctest, run_unittest, cpython_only @@ -118,44 +134,6 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) - def dump(self, co): - dump = {} - for attr in ["name", "argcount", "kwonlyargcount", "names", "varnames", - "cellvars", "freevars", "nlocals", "flags"]: - dump[attr] = getattr(co, "co_" + attr) - dump['consts'] = tuple(consts(co.co_consts)) - return dump - - def test_optimize_away(self): - ns = {} - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=SyntaxWarning) - exec(textwrap.dedent(''' - def optimize_away(): - 'doc string' - 'not a docstring' - 53 - 0x53 - b'bytes' - 1.0 - True - False - None - ... - '''), ns) - - self.assertEqual(self.dump(ns['optimize_away'].__code__), - {'name': 'optimize_away', - 'argcount': 0, - 'kwonlyargcount': 0, - 'names': (), - 'varnames': (), - 'cellvars': (), - 'freevars': (), - 'nlocals': 0, - 'flags': 67, - 'consts': ("'doc string'", 'None')}) - class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 2d6f5ed..8f8d71c 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -7,7 +7,6 @@ import unittest import sys # testing import * from sys import * -from test import support class TokenTests(unittest.TestCase): @@ -425,11 +424,8 @@ class GrammarTests(unittest.TestCase): # Tested below def test_expr_stmt(self): - msg = 'ignore constant statement' - with support.check_warnings((msg, SyntaxWarning)): - exec("1") - # (exprlist '=')* exprlist + 1 1, 2, 3 x = 1 x = 1, 2, 3 diff --git a/Misc/NEWS b/Misc/NEWS index e1875c0..d60aeb6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,9 +10,10 @@ Release date: tba Core and Builtins ----------------- -- Issue #26204: The compiler now ignores constant statements (ex: "def f(): 1") - and emit a SyntaxWarning warning. The warning is not emitted for string and - ellipsis (...) statements. +- Issue #26204: The compiler now ignores all constant statements: bytes, str, + int, float, complex, name constants (None, False, True), Ellipsis + and ast.Constant; not only str and int. For example, ``1.0`` is now ignored + in ``def f(): 1.0``. - Issue #4806: Avoid masking the original TypeError exception when using star (*) unpacking in function calls. Based on patch by Hagen Fürstenau and diff --git a/Python/compile.c b/Python/compile.c index 84b79a2..ca1d865 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2619,33 +2619,13 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value) switch (value->kind) { case Str_kind: + case Num_kind: case Ellipsis_kind: - /* Issue #26204: ignore string statement, but don't emit a - * SyntaxWarning. Triple quoted strings is a common syntax for - * multiline comments. - * - * Don't emit warning on "def f(): ..." neither. It's a legit syntax - * for abstract function. */ - return 1; - case Bytes_kind: - case Num_kind: case NameConstant_kind: case Constant_kind: - { - PyObject *msg = PyUnicode_FromString("ignore constant statement"); - if (msg == NULL) - return 0; - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, - msg, - c->c_filename, c->u->u_lineno, - NULL, NULL) == -1) { - Py_DECREF(msg); - return 0; - } - Py_DECREF(msg); + /* ignore constant statement */ return 1; - } default: break; -- cgit v0.12 From 38418662e0339b6331e0c7e00ea55f41b28bf38d Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 8 Feb 2016 20:34:49 -0800 Subject: Issue #26200: The SETREF macro adds unnecessary work in some cases. --- Modules/_collectionsmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 479b052..0b7a88f 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1218,6 +1218,7 @@ deque_del_item(dequeobject *deque, Py_ssize_t i) static int deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) { + PyObject *old_value; block *b; Py_ssize_t n, len=Py_SIZE(deque), halflen=(len+1)>>1, index=i; @@ -1246,7 +1247,9 @@ deque_ass_item(dequeobject *deque, Py_ssize_t i, PyObject *v) b = b->leftlink; } Py_INCREF(v); - Py_SETREF(b->data[i], v); + old_value = b->data[i]; + b->data[i] = v; + Py_DECREF(old_value); return 0; } -- cgit v0.12 From 1fe0d13d1246b5d60a4f4fb8c627babd0e94ab87 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 10:06:36 +0000 Subject: Issue #26243: zlib.compress() keyword argument support by Aviv Palivoda --- Doc/library/zlib.rst | 11 ++++++++--- Doc/whatsnew/3.6.rst | 7 +++++++ Lib/test/test_zlib.py | 4 ++++ Misc/NEWS | 3 +++ Modules/clinic/zlibmodule.c.h | 27 ++++++++++++++------------- Modules/zlibmodule.c | 17 ++++++++--------- 6 files changed, 44 insertions(+), 25 deletions(-) diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 1869bb8..09026cb 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -46,14 +46,19 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data[, level]) +.. function:: compress(data, level=-1) Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` controlling the level of compression; + *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``6``. + produces the most. ``0`` is no compression. The default value is ``-1`` + (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. + .. versionchanged:: 3.6 + Keyword arguments are now supported. + .. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b7cc159..87199df 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -150,6 +150,13 @@ to check if the :class:`~zipfile.ZipInfo` instance represents a directory. (Contributed by Thomas Kluyver in :issue:`26039`.) +zlib +---- + +The :func:`~zlib.compress` function now accepts keyword arguments. +(Contributed by Aviv Palivoda in :issue:`26243`.) + + Optimizations ============= diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index ecdb5a7..ca30116 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -162,6 +162,10 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase): x = zlib.compress(HAMLET_SCENE) self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_keywords(self): + x = zlib.compress(data=HAMLET_SCENE, level=3) + self.assertEqual(zlib.decompress(x), HAMLET_SCENE) + def test_speech128(self): # compress more data data = HAMLET_SCENE * 128 diff --git a/Misc/NEWS b/Misc/NEWS index d308762..6ce7e2d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -175,6 +175,9 @@ Core and Builtins Library ------- +- Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv + Palivoda. + - Issue #26117: The os.scandir() iterator now closes file descriptor not only when the iteration is finished, but when it was failed with error. diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 2d75bc9..d92f88b 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -3,38 +3,39 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(zlib_compress__doc__, -"compress($module, bytes, level=Z_DEFAULT_COMPRESSION, /)\n" +"compress($module, /, data, level=Z_DEFAULT_COMPRESSION)\n" "--\n" "\n" "Returns a bytes object containing compressed data.\n" "\n" -" bytes\n" +" data\n" " Binary data to be compressed.\n" " level\n" " Compression level, in 0-9."); #define ZLIB_COMPRESS_METHODDEF \ - {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__}, + {"compress", (PyCFunction)zlib_compress, METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__}, static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level); +zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level); static PyObject * -zlib_compress(PyModuleDef *module, PyObject *args) +zlib_compress(PyModuleDef *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - Py_buffer bytes = {NULL, NULL}; + static char *_keywords[] = {"data", "level", NULL}; + Py_buffer data = {NULL, NULL}; int level = Z_DEFAULT_COMPRESSION; - if (!PyArg_ParseTuple(args, "y*|i:compress", - &bytes, &level)) + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|i:compress", _keywords, + &data, &level)) goto exit; - return_value = zlib_compress_impl(module, &bytes, level); + return_value = zlib_compress_impl(module, &data, level); exit: - /* Cleanup for bytes */ - if (bytes.obj) - PyBuffer_Release(&bytes); + /* Cleanup for data */ + if (data.obj) + PyBuffer_Release(&data); return return_value; } @@ -439,4 +440,4 @@ exit: #ifndef ZLIB_COMPRESS_COPY_METHODDEF #define ZLIB_COMPRESS_COPY_METHODDEF #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */ -/*[clinic end generated code: output=cf81e1deae3af0ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3c96b58b923c1273 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 5cdab45..8ddc774 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -137,18 +137,17 @@ PyZlib_Free(voidpf ctx, void *ptr) /*[clinic input] zlib.compress - bytes: Py_buffer + data: Py_buffer Binary data to be compressed. level: int(c_default="Z_DEFAULT_COMPRESSION") = Z_DEFAULT_COMPRESSION - Compression level, in 0-9. - / + Compression level, in 0-9 or -1. Returns a bytes object containing compressed data. [clinic start generated code]*/ static PyObject * -zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level) -/*[clinic end generated code: output=5d7dd4588788efd3 input=be3abe9934bda4b3]*/ +zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level) +/*[clinic end generated code: output=1b97589132b203b4 input=671c615a4b2267da]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; @@ -156,13 +155,13 @@ zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int level) int err; z_stream zst; - if ((size_t)bytes->len > UINT_MAX) { + if ((size_t)data->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error; } - input = bytes->buf; - length = (unsigned int)bytes->len; + input = data->buf; + length = (unsigned int)data->len; zst.avail_out = length + length/1000 + 12 + 1; @@ -1323,7 +1322,7 @@ PyDoc_STRVAR(zlib_module_documentation, "zlib library, which is based on GNU zip.\n" "\n" "adler32(string[, start]) -- Compute an Adler-32 checksum.\n" -"compress(string[, level]) -- Compress string, with compression level in 0-9.\n" +"compress(data[, level]) -- Compress data, with compression level 0-9 or -1.\n" "compressobj([level[, ...]]) -- Return a compressor object.\n" "crc32(string[, start]) -- Compute a CRC-32 checksum.\n" "decompress(string,[wbits],[bufsize]) -- Decompresses a compressed string.\n" -- cgit v0.12 From b0cb42dfdb422495fb0c3a4374f2fa010f56a5ac Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 10:45:54 +0000 Subject: Issue 26243: Forgot to update zlib doc strings in Argument Clinic --- Modules/clinic/zlibmodule.c.h | 4 ++-- Modules/zlibmodule.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index d92f88b..222cb67 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -11,7 +11,7 @@ PyDoc_STRVAR(zlib_compress__doc__, " data\n" " Binary data to be compressed.\n" " level\n" -" Compression level, in 0-9."); +" Compression level, in 0-9 or -1."); #define ZLIB_COMPRESS_METHODDEF \ {"compress", (PyCFunction)zlib_compress, METH_VARARGS|METH_KEYWORDS, zlib_compress__doc__}, @@ -440,4 +440,4 @@ exit: #ifndef ZLIB_COMPRESS_COPY_METHODDEF #define ZLIB_COMPRESS_COPY_METHODDEF #endif /* !defined(ZLIB_COMPRESS_COPY_METHODDEF) */ -/*[clinic end generated code: output=3c96b58b923c1273 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e6f3b79e051ecc35 input=a9049054013a1b77]*/ diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 8ddc774..db3d8c9 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -147,7 +147,7 @@ Returns a bytes object containing compressed data. static PyObject * zlib_compress_impl(PyModuleDef *module, Py_buffer *data, int level) -/*[clinic end generated code: output=1b97589132b203b4 input=671c615a4b2267da]*/ +/*[clinic end generated code: output=1b97589132b203b4 input=abed30f4fa14e213]*/ { PyObject *ReturnVal = NULL; Byte *input, *output = NULL; -- cgit v0.12 From 7e3a91a5fc056f72c93f015bb9f4f22aef9e72fb Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 04:40:48 +0000 Subject: Issue #26136: Upgrade the generator_stop warning to DeprecationWarning Patch by Anish Shah. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/test/test_contextlib.py | 2 +- Lib/test/test_generators.py | 6 +++--- Lib/test/test_with.py | 4 ++-- Misc/NEWS | 4 ++++ Objects/genobject.c | 2 +- 6 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 87199df..3822d01 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -234,6 +234,14 @@ Deprecated features (Contributed by Rose Ames in :issue:`25791`.) +Deprecated Python behavior +-------------------------- + +* Raising the :exc:`StopIteration` exception inside a generator will now generate a + :exc:`DeprecationWarning`, and will trigger a :exc:`RuntimeError` in Python 3.7. + See :ref:`whatsnew-pep-479` for details. + + Removed ======= diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 30a6377..04fc875 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -89,7 +89,7 @@ class ContextManagerTestCase(unittest.TestCase): def woohoo(): yield try: - with self.assertWarnsRegex(PendingDeprecationWarning, + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with woohoo(): raise stop_exc diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index b92d5ce..0389b48 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -245,11 +245,11 @@ class ExceptionTest(unittest.TestCase): yield with self.assertRaises(StopIteration), \ - self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + self.assertWarnsRegex(DeprecationWarning, "StopIteration"): next(gen()) - with self.assertRaisesRegex(PendingDeprecationWarning, + with self.assertRaisesRegex(DeprecationWarning, "generator .* raised StopIteration"), \ warnings.catch_warnings(): @@ -268,7 +268,7 @@ class ExceptionTest(unittest.TestCase): g = f() self.assertEqual(next(g), 1) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): with self.assertRaises(StopIteration): next(g) diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index e8d789b..3815062 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -454,7 +454,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise StopIteration("from with") - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedStopIteration2(self): @@ -482,7 +482,7 @@ class ExceptionalTestCase(ContextmanagerAssertionMixin, unittest.TestCase): with cm(): raise next(iter([])) - with self.assertWarnsRegex(PendingDeprecationWarning, "StopIteration"): + with self.assertWarnsRegex(DeprecationWarning, "StopIteration"): self.assertRaises(StopIteration, shouldThrow) def testRaisedGeneratorExit1(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6ce7e2d..7912790 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26136: Upgrade the warning when a generator raises StopIteration + from PendingDeprecationWarning to DeprecationWarning. Patch by Anish + Shah. + - Issue #26204: The compiler now ignores all constant statements: bytes, str, int, float, complex, name constants (None, False, True), Ellipsis and ast.Constant; not only str and int. For example, ``1.0`` is now ignored diff --git a/Objects/genobject.c b/Objects/genobject.c index 81a80b7..8ab7cf1 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -178,7 +178,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) /* Pop the exception before issuing a warning. */ PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_PendingDeprecationWarning, 1, + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "generator '%.50S' raised StopIteration", gen->gi_qualname)) { /* Warning was converted to an error. */ -- cgit v0.12 From 288ed038aa28b9037e862026d743d4010a615f4e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 10 Feb 2016 05:45:55 +0000 Subject: =?UTF-8?q?Issue=20#26304:=20Additional=20=E2=80=9Callows=20to?= =?UTF-8?q?=E2=80=9D=20fix=20specific=20to=203.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3822d01..bc5e550 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -144,7 +144,7 @@ zipfile ------- A new :meth:`ZipInfo.from_file() ` class method -allow to make :class:`~zipfile.ZipInfo` instance from a filesystem file. +allows making a :class:`~zipfile.ZipInfo` instance from a filesystem file. A new :meth:`ZipInfo.is_dir() ` method can be used to check if the :class:`~zipfile.ZipInfo` instance represents a directory. (Contributed by Thomas Kluyver in :issue:`26039`.) -- cgit v0.12 From 3fb5612e085bb9fd979057372978c0ad08c29226 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 10 Feb 2016 09:46:56 -0800 Subject: Hopefully clarify the difference between Optional[t] and an optional argument. --- Doc/library/typing.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 41f594e..12b5490 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -286,6 +286,13 @@ The module defines the following classes, functions and decorators: ``Optional[X]`` is equivalent to ``Union[X, type(None)]``. + Note that this is not the same concept as an optional argument, + which is one that has a default. An optional argument with a + default needn't use the ``Optional`` qualifier on its type + annotation (although it is inferred if the default is ``None``). + A mandatory argument may still have an ``Optional`` type if an + explicit value of ``None`` is allowed. + .. class:: Tuple Tuple type; ``Tuple[X, Y]`` is the is the type of a tuple of two items -- cgit v0.12 From 78f55ffc63ba637ffa4a07ca869f451e680b002a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 10 Feb 2016 22:58:18 +0000 Subject: Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. --- Lib/multiprocessing/pool.py | 20 ++++++++++++-------- Lib/test/_test_multiprocessing.py | 24 ++++++++++++++++++++++++ Misc/NEWS | 2 ++ 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 6d25469..ffdf426 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -638,22 +638,26 @@ class MapResult(ApplyResult): self._number_left = length//chunksize + bool(length % chunksize) def _set(self, i, success_result): + self._number_left -= 1 success, result = success_result - if success: + if success and self._success: self._value[i*self._chunksize:(i+1)*self._chunksize] = result - self._number_left -= 1 if self._number_left == 0: if self._callback: self._callback(self._value) del self._cache[self._job] self._event.set() else: - self._success = False - self._value = result - if self._error_callback: - self._error_callback(self._value) - del self._cache[self._job] - self._event.set() + if not success and self._success: + # only store first exception + self._success = False + self._value = result + if self._number_left == 0: + # only consider the result ready once all jobs are done + if self._error_callback: + self._error_callback(self._value) + del self._cache[self._job] + self._event.set() # # Class whose instances are returned by `Pool.imap()` diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e9120ab..a59d2ba 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1660,6 +1660,10 @@ def sqr(x, wait=0.0): def mul(x, y): return x*y +def raise_large_valuerror(wait): + time.sleep(wait) + raise ValueError("x" * 1024**2) + class SayWhenError(ValueError): pass def exception_throwing_generator(total, when): @@ -1895,6 +1899,26 @@ class _TestPool(BaseTestCase): with self.assertRaises(RuntimeError): p.apply(self._test_wrapped_exception) + def test_map_no_failfast(self): + # Issue #23992: the fail-fast behaviour when an exception is raised + # during map() would make Pool.join() deadlock, because a worker + # process would fill the result queue (after the result handler thread + # terminated, hence not draining it anymore). + + t_start = time.time() + + with self.assertRaises(ValueError): + with self.Pool(2) as p: + try: + p.map(raise_large_valuerror, [0, 1]) + finally: + time.sleep(0.5) + p.close() + p.join() + + # check that we indeed waited for all jobs + self.assertGreater(time.time() - t_start, 0.9) + def raising(): raise KeyError("key") diff --git a/Misc/NEWS b/Misc/NEWS index 1ac4ad0..22a392e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,8 @@ Core and Builtins Library ------- +- Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. + - Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv Palivoda. -- cgit v0.12 From a9725f86a984f74e74f09c4808fc8f4b403728b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 12:41:40 +0200 Subject: Issue #26312: SystemError is now raised in all programming bugs with using PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some programming bugs. --- Lib/test/test_capi.py | 4 ++-- Misc/NEWS | 4 ++++ Python/getargs.c | 12 ++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1a743fd..74ec6c5 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -491,7 +491,7 @@ class SkipitemTest(unittest.TestCase): except SystemError as e: s = "argument 1 (impossible)" when_not_skipped = (str(e) == s) - except (TypeError, RuntimeError): + except TypeError: when_not_skipped = False # test the format unit when skipped @@ -500,7 +500,7 @@ class SkipitemTest(unittest.TestCase): _testcapi.parse_tuple_and_keywords(empty_tuple, dict_b, optional_format.encode("ascii"), keywords) when_skipped = False - except RuntimeError as e: + except SystemError as e: s = "impossible: '{}'".format(format) when_skipped = (str(e) == s) diff --git a/Misc/NEWS b/Misc/NEWS index 22a392e..82b49d2 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -741,6 +741,10 @@ Tools/Demos C API ----- +- Issue #26312: SystemError is now raised in all programming bugs with using + PyArg_ParseTupleAndKeywords(). RuntimeError did raised before in some + programming bugs. + - Issue #26198: ValueError is now raised instead of TypeError on buffer overflow in parsing "es#" and "et#" format units. SystemError is now raised instead of TypeError on programmical error in parsing format string. diff --git a/Python/getargs.c b/Python/getargs.c index be6e375..66a0c00 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1512,7 +1512,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, keyword = kwlist[i]; if (*format == '|') { if (min != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string (| specified twice)"); return cleanreturn(0, &freelist); } @@ -1521,14 +1521,14 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, format++; if (max != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string ($ before |)"); return cleanreturn(0, &freelist); } } if (*format == '$') { if (max != INT_MAX) { - PyErr_SetString(PyExc_RuntimeError, + PyErr_SetString(PyExc_SystemError, "Invalid format string ($ specified twice)"); return cleanreturn(0, &freelist); } @@ -1546,7 +1546,7 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, } } if (IS_END_OF_FORMAT(*format)) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_SystemError, "More keyword list entries (%d) than " "format specifiers (%d)", len, i); return cleanreturn(0, &freelist); @@ -1598,14 +1598,14 @@ vgetargskeywords(PyObject *args, PyObject *keywords, const char *format, * keyword args */ msg = skipitem(&format, p_va, flags); if (msg) { - PyErr_Format(PyExc_RuntimeError, "%s: '%s'", msg, + PyErr_Format(PyExc_SystemError, "%s: '%s'", msg, format); return cleanreturn(0, &freelist); } } if (!IS_END_OF_FORMAT(*format) && (*format != '|') && (*format != '$')) { - PyErr_Format(PyExc_RuntimeError, + PyErr_Format(PyExc_SystemError, "more argument specifiers than keyword list entries " "(remaining format:'%s')", format); return cleanreturn(0, &freelist); -- cgit v0.12 From 885bdc4946890f4bb80557fab80c3874b2cc4d39 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:10:36 +0200 Subject: Issue #25985: sys.version_info is now used instead of sys.version to format short Python version. --- Doc/c-api/intro.rst | 7 ++++--- Doc/includes/test.py | 2 +- Doc/library/sysconfig.rst | 2 +- Lib/distutils/command/bdist_msi.py | 2 +- Lib/distutils/command/bdist_wininst.py | 2 +- Lib/distutils/command/build.py | 4 ++-- Lib/distutils/command/install.py | 4 ++-- Lib/distutils/command/install_egg_info.py | 4 ++-- Lib/distutils/sysconfig.py | 2 +- Lib/distutils/tests/test_build.py | 5 +++-- Lib/importlib/_bootstrap_external.py | 2 +- Lib/pydoc.py | 6 +++--- Lib/site.py | 4 ++-- Lib/sysconfig.py | 8 ++++---- Lib/test/test_importlib/test_windows.py | 2 +- Lib/test/test_site.py | 5 +++-- Lib/test/test_venv.py | 2 +- Lib/urllib/request.py | 2 +- Lib/xmlrpc/client.py | 2 +- Makefile.pre.in | 2 +- Python/importlib_external.h | 31 ++++++++++++++++--------------- Tools/freeze/freeze.py | 2 +- Tools/scripts/nm2def.py | 4 ++-- setup.py | 2 +- 24 files changed, 56 insertions(+), 52 deletions(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 95cbef5..74681d2 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -64,9 +64,10 @@ The header files are typically installed with Python. On Unix, these are located in the directories :file:`{prefix}/include/pythonversion/` and :file:`{exec_prefix}/include/pythonversion/`, where :envvar:`prefix` and :envvar:`exec_prefix` are defined by the corresponding parameters to Python's -:program:`configure` script and *version* is ``sys.version[:3]``. On Windows, -the headers are installed in :file:`{prefix}/include`, where :envvar:`prefix` is -the installation directory specified to the installer. +:program:`configure` script and *version* is +``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed +in :file:`{prefix}/include`, where :envvar:`prefix` is the installation +directory specified to the installer. To include the headers, place both directories (if different) on your compiler's search path for includes. Do *not* place the parent directories on the search diff --git a/Doc/includes/test.py b/Doc/includes/test.py index 7ebf46a..9e9d4a6 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/test.py @@ -204,7 +204,7 @@ Test cyclic gc(?) import os import sys from distutils.util import get_platform -PLAT_SPEC = "%s-%s" % (get_platform(), sys.version[0:3]) +PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) src = os.path.join("build", "lib.%s" % PLAT_SPEC) sys.path.append(src) diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 535ac54..47f3900 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -163,7 +163,7 @@ Other functions .. function:: get_python_version() Return the ``MAJOR.MINOR`` Python version number as a string. Similar to - ``sys.version[:3]``. + ``'%d.%d' % sys.version_info[:2]``. .. function:: get_platform() diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py index b3cfe9c..f6c21ae 100644 --- a/Lib/distutils/command/bdist_msi.py +++ b/Lib/distutils/command/bdist_msi.py @@ -199,7 +199,7 @@ class bdist_msi(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py index 0c0e2c1..d3e1d3a 100644 --- a/Lib/distutils/command/bdist_wininst.py +++ b/Lib/distutils/command/bdist_wininst.py @@ -141,7 +141,7 @@ class bdist_wininst(Command): target_version = self.target_version if not target_version: assert self.skip_build, "Should have already checked this" - target_version = sys.version[0:3] + target_version = '%d.%d' % sys.version_info[:2] plat_specifier = ".%s-%s" % (self.plat_name, target_version) build = self.get_finalized_command('build') build.build_lib = os.path.join(build.build_base, diff --git a/Lib/distutils/command/build.py b/Lib/distutils/command/build.py index 337dd0b..c6f52e6 100644 --- a/Lib/distutils/command/build.py +++ b/Lib/distutils/command/build.py @@ -81,7 +81,7 @@ class build(Command): "--plat-name only supported on Windows (try " "using './configure --help' on your platform)") - plat_specifier = ".%s-%s" % (self.plat_name, sys.version[0:3]) + plat_specifier = ".%s-%d.%d" % (self.plat_name, *sys.version_info[:2]) # Make it so Python 2.x and Python 2.x with --with-pydebug don't # share the same build directories. Doing so confuses the build @@ -114,7 +114,7 @@ class build(Command): 'temp' + plat_specifier) if self.build_scripts is None: self.build_scripts = os.path.join(self.build_base, - 'scripts-' + sys.version[0:3]) + 'scripts-%d.%d' % sys.version_info[:2]) if self.executable is None: self.executable = os.path.normpath(sys.executable) diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 67db007..9474e9c 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -290,8 +290,8 @@ class install(Command): 'dist_version': self.distribution.get_version(), 'dist_fullname': self.distribution.get_fullname(), 'py_version': py_version, - 'py_version_short': py_version[0:3], - 'py_version_nodot': py_version[0] + py_version[2], + 'py_version_short': '%d.%d' % sys.version_info[:2], + 'py_version_nodot': '%d%d' % sys.version_info[:2], 'sys_prefix': prefix, 'prefix': prefix, 'sys_exec_prefix': exec_prefix, diff --git a/Lib/distutils/command/install_egg_info.py b/Lib/distutils/command/install_egg_info.py index c2a7d64..0ddc736 100644 --- a/Lib/distutils/command/install_egg_info.py +++ b/Lib/distutils/command/install_egg_info.py @@ -21,10 +21,10 @@ class install_egg_info(Command): def finalize_options(self): self.set_undefined_options('install_lib',('install_dir','install_dir')) - basename = "%s-%s-py%s.egg-info" % ( + basename = "%s-%s-py%d.%d.egg-info" % ( to_filename(safe_name(self.distribution.get_name())), to_filename(safe_version(self.distribution.get_version())), - sys.version[:3] + *sys.version_info[:2] ) self.target = os.path.join(self.install_dir, basename) self.outputs = [self.target] diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 573724d..d203f8e 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -70,7 +70,7 @@ def get_python_version(): leaving off the patchlevel. Sample return values could be '1.5' or '2.2'. """ - return sys.version[:3] + return '%d.%d' % sys.version_info[:2] def get_python_inc(plat_specific=0, prefix=None): diff --git a/Lib/distutils/tests/test_build.py b/Lib/distutils/tests/test_build.py index 3391f36..b020a5b 100644 --- a/Lib/distutils/tests/test_build.py +++ b/Lib/distutils/tests/test_build.py @@ -27,7 +27,7 @@ class BuildTestCase(support.TempdirManager, # build_platlib is 'build/lib.platform-x.x[-pydebug]' # examples: # build/lib.macosx-10.3-i386-2.7 - plat_spec = '.%s-%s' % (cmd.plat_name, sys.version[0:3]) + plat_spec = '.%s-%d.%d' % (cmd.plat_name, *sys.version_info[:2]) if hasattr(sys, 'gettotalrefcount'): self.assertTrue(cmd.build_platlib.endswith('-pydebug')) plat_spec += '-pydebug' @@ -42,7 +42,8 @@ class BuildTestCase(support.TempdirManager, self.assertEqual(cmd.build_temp, wanted) # build_scripts is build/scripts-x.x - wanted = os.path.join(cmd.build_base, 'scripts-' + sys.version[0:3]) + wanted = os.path.join(cmd.build_base, + 'scripts-%d.%d' % sys.version_info[:2]) self.assertEqual(cmd.build_scripts, wanted) # executable is os.path.normpath(sys.executable) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 71098f1..ddb2456 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -593,7 +593,7 @@ class WindowsRegistryFinder: else: registry_key = cls.REGISTRY_KEY key = registry_key.format(fullname=fullname, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with cls._open_registry(key) as hkey: filepath = _winreg.QueryValue(hkey, '') diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a73298d..5e5a8ae 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1911,10 +1911,10 @@ has the same effect as typing a particular string at the help> prompt. def intro(self): self.output.write(''' -Welcome to Python %s's help utility! +Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out -the tutorial on the Internet at http://docs.python.org/%s/tutorial/. +the tutorial on the Internet at http://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and @@ -1924,7 +1924,7 @@ To get a list of available modules, keywords, symbols, or topics, type "modules", "keywords", "symbols", or "topics". Each module also comes with a one-line summary of what it does; to list the modules whose name or summary contain a given string such as "spam", type "modules spam". -''' % tuple([sys.version[:3]]*2)) +'''.format('%d.%d' % sys.version_info[:2])) def list(self, items, columns=4, width=80): items = list(sorted(items)) diff --git a/Lib/site.py b/Lib/site.py index 13ec8fa..56ba709 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -304,7 +304,7 @@ def getsitepackages(prefixes=None): if os.sep == '/': sitepackages.append(os.path.join(prefix, "lib", - "python" + sys.version[:3], + "python%d.%d" % sys.version_info[:2], "site-packages")) else: sitepackages.append(prefix) @@ -317,7 +317,7 @@ def getsitepackages(prefixes=None): if framework: sitepackages.append( os.path.join("/Library", framework, - sys.version[:3], "site-packages")) + '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 9c34be0..f18b1bc 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -86,8 +86,8 @@ _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', # FIXME don't rely on sys.version here, its format is an implementation detail # of CPython, use sys.version_info or sys.hexversion _PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = sys.version[:3] -_PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] +_PY_VERSION_SHORT = '%d.%d' % sys.version_info[:2] +_PY_VERSION_SHORT_NO_DOT = '%d%d' % sys.version_info[:2] _PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) @@ -386,7 +386,7 @@ def _generate_posix_vars(): module.build_time_vars = vars sys.modules[name] = module - pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT) if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) @@ -518,7 +518,7 @@ def get_config_vars(*args): _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION[0] + _PY_VERSION[2] + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT _CONFIG_VARS['installed_base'] = _BASE_PREFIX _CONFIG_VARS['base'] = _PREFIX _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index c893bcf..005b685 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -40,7 +40,7 @@ def setup_module(machinery, name, path=None): else: root = machinery.WindowsRegistryFinder.REGISTRY_KEY key = root.format(fullname=name, - sys_version=sys.version[:3]) + sys_version='%d.%d' % sys.version_info[:2]) try: with temp_module(name, "a = 1") as location: subkey = CreateKey(HKEY_CURRENT_USER, key) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 6615080..574e496 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -238,13 +238,14 @@ class HelperFunctionsTests(unittest.TestCase): self.assertEqual(len(dirs), 2) wanted = os.path.join('/Library', sysconfig.get_config_var("PYTHONFRAMEWORK"), - sys.version[:3], + '%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[1], wanted) elif os.sep == '/': # OS X non-framwework builds, Linux, FreeBSD, etc self.assertEqual(len(dirs), 1) - wanted = os.path.join('xoxo', 'lib', 'python' + sys.version[:3], + wanted = os.path.join('xoxo', 'lib', + 'python%d.%d' % sys.version_info[:2], 'site-packages') self.assertEqual(dirs[0], wanted) else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 28b0f6c..f38f52df 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -51,7 +51,7 @@ class BaseTest(unittest.TestCase): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%s' % sys.version[:3]) + self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: executable = os.environ['__PYVENV_LAUNCHER__'] diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 4c2b9fe..e3eed16 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -133,7 +133,7 @@ __all__ = [ ] # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] _opener = None def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index bf42835..07a4f03 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -151,7 +151,7 @@ def escape(s): return s.replace(">", ">",) # used in User-Agent header sent -__version__ = sys.version[:3] +__version__ = '%d.%d' % sys.version_info[:2] # xmlrpc integer limits MAXINT = 2**31-1 diff --git a/Makefile.pre.in b/Makefile.pre.in index 2a687e5..6514bf8 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -568,7 +568,7 @@ $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) platform: $(BUILDPYTHON) pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print(get_platform()+"-"+sys.version[0:3])' >platform + $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform # Create build directory and generate the sysconfig build-time data there. # pybuilddir.txt contains the name of the build dir and is used for diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 97c4948..3d7aff0 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -888,22 +888,23 @@ const unsigned char _Py_M__importlib_external[] = { 87,105,110,100,111,119,115,82,101,103,105,115,116,114,121,70, 105,110,100,101,114,46,95,111,112,101,110,95,114,101,103,105, 115,116,114,121,99,2,0,0,0,0,0,0,0,6,0,0, - 0,16,0,0,0,67,0,0,0,115,143,0,0,0,124,0, + 0,16,0,0,0,67,0,0,0,115,147,0,0,0,124,0, 0,106,0,0,114,21,0,124,0,0,106,1,0,125,2,0, 110,9,0,124,0,0,106,2,0,125,2,0,124,2,0,106, - 3,0,100,1,0,124,1,0,100,2,0,116,4,0,106,5, - 0,100,0,0,100,3,0,133,2,0,25,131,0,2,125,3, - 0,121,47,0,124,0,0,106,6,0,124,3,0,131,1,0, - 143,25,0,125,4,0,116,7,0,106,8,0,124,4,0,100, - 4,0,131,2,0,125,5,0,87,100,0,0,81,82,88,87, - 110,22,0,4,116,9,0,107,10,0,114,138,0,1,1,1, - 100,0,0,83,89,110,1,0,88,124,5,0,83,41,5,78, - 114,119,0,0,0,90,11,115,121,115,95,118,101,114,115,105, - 111,110,114,80,0,0,0,114,30,0,0,0,41,10,218,11, - 68,69,66,85,71,95,66,85,73,76,68,218,18,82,69,71, - 73,83,84,82,89,95,75,69,89,95,68,69,66,85,71,218, - 12,82,69,71,73,83,84,82,89,95,75,69,89,114,47,0, - 0,0,114,7,0,0,0,218,7,118,101,114,115,105,111,110, + 3,0,100,1,0,124,1,0,100,2,0,100,3,0,116,4, + 0,106,5,0,100,0,0,100,4,0,133,2,0,25,22,131, + 0,2,125,3,0,121,47,0,124,0,0,106,6,0,124,3, + 0,131,1,0,143,25,0,125,4,0,116,7,0,106,8,0, + 124,4,0,100,5,0,131,2,0,125,5,0,87,100,0,0, + 81,82,88,87,110,22,0,4,116,9,0,107,10,0,114,142, + 0,1,1,1,100,0,0,83,89,110,1,0,88,124,5,0, + 83,41,6,78,114,119,0,0,0,90,11,115,121,115,95,118, + 101,114,115,105,111,110,122,5,37,100,46,37,100,114,56,0, + 0,0,114,30,0,0,0,41,10,218,11,68,69,66,85,71, + 95,66,85,73,76,68,218,18,82,69,71,73,83,84,82,89, + 95,75,69,89,95,68,69,66,85,71,218,12,82,69,71,73, + 83,84,82,89,95,75,69,89,114,47,0,0,0,114,7,0, + 0,0,218,12,118,101,114,115,105,111,110,95,105,110,102,111, 114,166,0,0,0,114,163,0,0,0,90,10,81,117,101,114, 121,86,97,108,117,101,114,40,0,0,0,41,6,114,164,0, 0,0,114,119,0,0,0,90,12,114,101,103,105,115,116,114, @@ -911,7 +912,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,8,102,105,108,101,112,97,116,104,114,4,0,0,0,114, 4,0,0,0,114,5,0,0,0,218,16,95,115,101,97,114, 99,104,95,114,101,103,105,115,116,114,121,77,2,0,0,115, - 22,0,0,0,0,2,9,1,12,2,9,1,15,1,22,1, + 22,0,0,0,0,2,9,1,12,2,9,1,15,1,26,1, 3,1,18,1,29,1,13,1,9,1,122,38,87,105,110,100, 111,119,115,82,101,103,105,115,116,114,121,70,105,110,100,101, 114,46,95,115,101,97,114,99,104,95,114,101,103,105,115,116, diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index c075807..389fffd 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -218,7 +218,7 @@ def main(): ishome = os.path.exists(os.path.join(prefix, 'Python', 'ceval.c')) # locations derived from options - version = sys.version[:3] + version = '%d.%d' % sys.version_info[:2] flagged_version = version + sys.abiflags if win: extensions_c = 'frozen_extensions.c' diff --git a/Tools/scripts/nm2def.py b/Tools/scripts/nm2def.py index 8f07559..83bbcd7 100755 --- a/Tools/scripts/nm2def.py +++ b/Tools/scripts/nm2def.py @@ -36,8 +36,8 @@ option to produce this format (since it is the original v7 Unix format). """ import os, sys -PYTHONLIB = 'libpython'+sys.version[:3]+'.a' -PC_PYTHONLIB = 'Python'+sys.version[0]+sys.version[2]+'.dll' +PYTHONLIB = 'libpython%d.%d.a' % sys.version_info[:2] +PC_PYTHONLIB = 'Python%d%d.dll' % sys.version_info[:2] NM = 'nm -p -g %s' # For Linux, use "nm -g %s" def symbols(lib=PYTHONLIB,types=('T','C','D')): diff --git a/setup.py b/setup.py index 930b64a..720f568 100644 --- a/setup.py +++ b/setup.py @@ -2226,7 +2226,7 @@ def main(): setup(# PyPI Metadata (PEP 301) name = "Python", version = sys.version.split()[0], - url = "http://www.python.org/%s" % sys.version[:3], + url = "http://www.python.org/%d.%d" % sys.version_info[:2], maintainer = "Guido van Rossum and the Python community", maintainer_email = "python-dev@python.org", description = "A high-level object-oriented programming language", -- cgit v0.12 From ffe96ae10be8a3117fa18c35034fcfc45c3cf7b7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:21:30 +0200 Subject: Issue #25994: Added the close() method and the support of the context manager protocol for the os.scandir() iterator. --- Doc/library/os.rst | 27 +++++++++++++-- Doc/whatsnew/3.6.rst | 11 ++++++ Lib/os.py | 94 ++++++++++++++++++++++++++++++--------------------- Lib/test/test_os.py | 52 ++++++++++++++++++++++++++++ Misc/NEWS | 3 ++ Modules/posixmodule.c | 73 +++++++++++++++++++++++++++++++++++---- 6 files changed, 211 insertions(+), 49 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f16a3fb..f064717 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1891,14 +1891,29 @@ features: :attr:`~DirEntry.path` attributes of each :class:`DirEntry` will be of the same type as *path*. + The :func:`scandir` iterator supports the :term:`context manager` protocol + and has the following method: + + .. method:: scandir.close() + + Close the iterator and free acquired resources. + + This is called automatically when the iterator is exhausted or garbage + collected, or when an error happens during iterating. However it + is advisable to call it explicitly or use the :keyword:`with` + statement. + + .. versionadded:: 3.6 + The following example shows a simple use of :func:`scandir` to display all the files (excluding directories) in the given *path* that don't start with ``'.'``. The ``entry.is_file()`` call will generally not make an additional system call:: - for entry in os.scandir(path): - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) + with os.scandir(path) as it: + for entry in it: + if not entry.name.startswith('.') and entry.is_file(): + print(entry.name) .. note:: @@ -1914,6 +1929,12 @@ features: .. versionadded:: 3.5 + .. versionadded:: 3.6 + Added support for the :term:`context manager` protocol and the + :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither + exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted + in its destructor. + .. class:: DirEntry diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bc5e550..b7d14f2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -104,6 +104,17 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) +os +-- + +A new :meth:`~os.scandir.close` method allows explicitly closing a +:func:`~os.scandir` iterator. The :func:`~os.scandir` iterator now +supports the :term:`context manager` protocol. If a :func:`scandir` +iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` +will be emitted in its destructor. +(Contributed by Serhiy Storchaka in :issue:`25994`.) + + pickle ------ diff --git a/Lib/os.py b/Lib/os.py index 674a7d7..c3f674e 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -374,46 +374,47 @@ def walk(top, topdown=True, onerror=None, followlinks=False): onerror(error) return - while True: - try: + with scandir_it: + while True: try: - entry = next(scandir_it) - except StopIteration: - break - except OSError as error: - if onerror is not None: - onerror(error) - return - - try: - is_dir = entry.is_dir() - except OSError: - # If is_dir() raises an OSError, consider that the entry is not - # a directory, same behaviour than os.path.isdir(). - is_dir = False - - if is_dir: - dirs.append(entry.name) - else: - nondirs.append(entry.name) + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + return - if not topdown and is_dir: - # Bottom-up: recurse into sub-directory, but exclude symlinks to - # directories if followlinks is False - if followlinks: - walk_into = True + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider that the entry is not + # a directory, same behaviour than os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) else: - try: - is_symlink = entry.is_symlink() - except OSError: - # If is_symlink() raises an OSError, consider that the - # entry is not a symbolic link, same behaviour than - # os.path.islink(). - is_symlink = False - walk_into = not is_symlink + nondirs.append(entry.name) - if walk_into: - yield from walk(entry.path, topdown, onerror, followlinks) + if not topdown and is_dir: + # Bottom-up: recurse into sub-directory, but exclude symlinks to + # directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider that the + # entry is not a symbolic link, same behaviour than + # os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + yield from walk(entry.path, topdown, onerror, followlinks) # Yield before recursion if going top down if topdown: @@ -437,15 +438,30 @@ class _DummyDirEntry: def __init__(self, dir, name): self.name = name self.path = path.join(dir, name) + def is_dir(self): return path.isdir(self.path) + def is_symlink(self): return path.islink(self.path) -def _dummy_scandir(dir): +class _dummy_scandir: # listdir-based implementation for bytes patches on Windows - for name in listdir(dir): - yield _DummyDirEntry(dir, name) + def __init__(self, dir): + self.dir = dir + self.it = iter(listdir(dir)) + + def __iter__(self): + return self + + def __next__(self): + return _DummyDirEntry(self.dir, next(self.it)) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.it = iter(()) __all__.append("walk") diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 66a426a..07682f2 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2808,6 +2808,8 @@ class ExportsTests(unittest.TestCase): class TestScandir(unittest.TestCase): + check_no_resource_warning = support.check_no_resource_warning + def setUp(self): self.path = os.path.realpath(support.TESTFN) self.addCleanup(support.rmtree, self.path) @@ -3030,6 +3032,56 @@ class TestScandir(unittest.TestCase): for obj in [1234, 1.234, {}, []]: self.assertRaises(TypeError, os.scandir, obj) + def test_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + iterator.close() + # multiple closes + iterator.close() + with self.check_no_resource_warning(): + del iterator + + def test_context_manager(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + with self.check_no_resource_warning(): + del iterator + + def test_context_manager_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + iterator.close() + + def test_context_manager_exception(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with self.assertRaises(ZeroDivisionError): + with os.scandir(self.path) as iterator: + next(iterator) + 1/0 + with self.check_no_resource_warning(): + del iterator + + def test_resource_warning(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + with self.assertWarns(ResourceWarning): + del iterator + support.gc_collect() + # exhausted iterator + iterator = os.scandir(self.path) + list(iterator) + with self.check_no_resource_warning(): + del iterator + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 405e0b9..92a210f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,9 @@ Core and Builtins Library ------- +- Issue #25994: Added the close() method and the support of the context manager + protocol for the os.scandir() iterator. + - Issue #23992: multiprocessing: make MapResult not fail-fast upon exception. - Issue #26243: Support keyword arguments to zlib.compress(). Patch by Aviv diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2688cbc..6e0081d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11937,8 +11937,14 @@ typedef struct { #ifdef MS_WINDOWS +static int +ScandirIterator_is_closed(ScandirIterator *iterator) +{ + return iterator->handle == INVALID_HANDLE_VALUE; +} + static void -ScandirIterator_close(ScandirIterator *iterator) +ScandirIterator_closedir(ScandirIterator *iterator) { if (iterator->handle == INVALID_HANDLE_VALUE) return; @@ -11956,7 +11962,7 @@ ScandirIterator_iternext(ScandirIterator *iterator) BOOL success; PyObject *entry; - /* Happens if the iterator is iterated twice */ + /* Happens if the iterator is iterated twice, or closed explicitly */ if (iterator->handle == INVALID_HANDLE_VALUE) return NULL; @@ -11987,14 +11993,20 @@ ScandirIterator_iternext(ScandirIterator *iterator) } /* Error or no more files */ - ScandirIterator_close(iterator); + ScandirIterator_closedir(iterator); return NULL; } #else /* POSIX */ +static int +ScandirIterator_is_closed(ScandirIterator *iterator) +{ + return !iterator->dirp; +} + static void -ScandirIterator_close(ScandirIterator *iterator) +ScandirIterator_closedir(ScandirIterator *iterator) { if (!iterator->dirp) return; @@ -12014,7 +12026,7 @@ ScandirIterator_iternext(ScandirIterator *iterator) int is_dot; PyObject *entry; - /* Happens if the iterator is iterated twice */ + /* Happens if the iterator is iterated twice, or closed explicitly */ if (!iterator->dirp) return NULL; @@ -12051,21 +12063,67 @@ ScandirIterator_iternext(ScandirIterator *iterator) } /* Error or no more files */ - ScandirIterator_close(iterator); + ScandirIterator_closedir(iterator); return NULL; } #endif +static PyObject * +ScandirIterator_close(ScandirIterator *self, PyObject *args) +{ + ScandirIterator_closedir(self); + Py_RETURN_NONE; +} + +static PyObject * +ScandirIterator_enter(PyObject *self, PyObject *args) +{ + Py_INCREF(self); + return self; +} + +static PyObject * +ScandirIterator_exit(ScandirIterator *self, PyObject *args) +{ + ScandirIterator_closedir(self); + Py_RETURN_NONE; +} + static void ScandirIterator_dealloc(ScandirIterator *iterator) { - ScandirIterator_close(iterator); + if (!ScandirIterator_is_closed(iterator)) { + PyObject *exc, *val, *tb; + Py_ssize_t old_refcount = Py_REFCNT(iterator); + /* Py_INCREF/Py_DECREF cannot be used, because the refcount is + * likely zero, Py_DECREF would call again the destructor. + */ + ++Py_REFCNT(iterator); + PyErr_Fetch(&exc, &val, &tb); + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed scandir iterator %R", iterator)) { + /* Spurious errors can appear at shutdown */ + if (PyErr_ExceptionMatches(PyExc_Warning)) + PyErr_WriteUnraisable((PyObject *) iterator); + } + PyErr_Restore(exc, val, tb); + Py_REFCNT(iterator) = old_refcount; + + ScandirIterator_closedir(iterator); + } Py_XDECREF(iterator->path.object); path_cleanup(&iterator->path); Py_TYPE(iterator)->tp_free((PyObject *)iterator); } +static PyMethodDef ScandirIterator_methods[] = { + {"__enter__", (PyCFunction)ScandirIterator_enter, METH_NOARGS}, + {"__exit__", (PyCFunction)ScandirIterator_exit, METH_VARARGS}, + {"close", (PyCFunction)ScandirIterator_close, METH_NOARGS}, + {NULL} +}; + static PyTypeObject ScandirIteratorType = { PyVarObject_HEAD_INIT(NULL, 0) MODNAME ".ScandirIterator", /* tp_name */ @@ -12095,6 +12153,7 @@ static PyTypeObject ScandirIteratorType = { 0, /* tp_weaklistoffset */ PyObject_SelfIter, /* tp_iter */ (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ + ScandirIterator_methods, /* tp_methods */ }; static PyObject * -- cgit v0.12 From 7c90a82a01aed150eb589b1c7035352e11cf8429 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 11 Feb 2016 13:31:00 +0200 Subject: Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. Different solution from 3.5. --- Lib/os.py | 6 +++++- Misc/NEWS | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/os.py b/Lib/os.py index c3f674e..0ea7b62 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -356,6 +356,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirs = [] nondirs = [] + walk_dirs = [] # We may not have read permission for top, in which case we can't # get a list of the files the directory contains. os.walk @@ -414,7 +415,7 @@ def walk(top, topdown=True, onerror=None, followlinks=False): walk_into = not is_symlink if walk_into: - yield from walk(entry.path, topdown, onerror, followlinks) + walk_dirs.append(entry.path) # Yield before recursion if going top down if topdown: @@ -431,6 +432,9 @@ def walk(top, topdown=True, onerror=None, followlinks=False): if followlinks or not islink(new_path): yield from walk(new_path, topdown, onerror, followlinks) else: + # Recurse into sub-directories + for new_path in walk_dirs: + yield from walk(new_path, topdown, onerror, followlinks) # Yield after recursion if going bottom up yield top, dirs, nondirs diff --git a/Misc/NEWS b/Misc/NEWS index 92a210f..0114a86 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -179,6 +179,8 @@ Core and Builtins Library ------- +- Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. + - Issue #25994: Added the close() method and the support of the context manager protocol for the os.scandir() iterator. -- cgit v0.12 From e0b23095ee03a11d09f38cbc689307dc5c93afda Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 11 Feb 2016 10:26:27 -0500 Subject: Issues #26289 and #26315: Optimize floor/modulo div for single-digit longs Microbenchmarks show 2-2.5x improvement. Built-in 'divmod' function is now also ~10% faster. -m timeit -s "x=22331" "x//2;x//-3;x//4;x//5;x//-6;x//7;x//8;x//-99;x//100;" with patch: 0.321 without patch: 0.633 -m timeit -s "x=22331" "x%2;x%3;x%-4;x%5;x%6;x%-7;x%8;x%99;x%-100;" with patch: 0.224 without patch: 0.66 Big thanks to Serhiy Storchaka, Mark Dickinson and Victor Stinner for thorow code reviews and algorithms improvements. --- Lib/test/test_long.py | 33 +++++++++++++++++++++ Misc/NEWS | 4 +++ Objects/longobject.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 62e69a9..5b8b4dc 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -689,6 +689,20 @@ class LongTest(unittest.TestCase): self.assertRaises(OverflowError, int, float('-inf')) self.assertRaises(ValueError, int, float('nan')) + def test_mod_division(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 % 0 + + self.assertEqual(13 % 10, 3) + self.assertEqual(-13 % 10, 7) + self.assertEqual(13 % -10, -7) + self.assertEqual(-13 % -10, -3) + + self.assertEqual(12 % 4, 0) + self.assertEqual(-12 % 4, 0) + self.assertEqual(12 % -4, 0) + self.assertEqual(-12 % -4, 0) + def test_true_division(self): huge = 1 << 40000 mhuge = -huge @@ -723,6 +737,25 @@ class LongTest(unittest.TestCase): for zero in ["huge / 0", "mhuge / 0"]: self.assertRaises(ZeroDivisionError, eval, zero, namespace) + def test_floordiv(self): + with self.assertRaises(ZeroDivisionError): + _ = 1 // 0 + + self.assertEqual(2 // 3, 0) + self.assertEqual(2 // -3, -1) + self.assertEqual(-2 // 3, -1) + self.assertEqual(-2 // -3, 0) + + self.assertEqual(-11 // -3, 3) + self.assertEqual(-11 // 3, -4) + self.assertEqual(11 // -3, -4) + self.assertEqual(11 // 3, 3) + + self.assertEqual(-12 // -3, 4) + self.assertEqual(-12 // 3, -4) + self.assertEqual(12 // -3, -4) + self.assertEqual(12 // 3, 4) + def check_truediv(self, a, b, skip_small=True): """Verify that the result of a/b is correctly rounded, by comparing it with a pure Python implementation of correctly diff --git a/Misc/NEWS b/Misc/NEWS index 0114a86..d51c3f3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -176,6 +176,10 @@ Core and Builtins - Issue #26288: Optimize PyLong_AsDouble. +- Issues #26289 and #26315: Optimize floor and modulo division for + single-digit longs. Microbenchmarks show 2-2.5x improvement. Built-in + 'divmod' function is now also ~10% faster. + Library ------- diff --git a/Objects/longobject.c b/Objects/longobject.c index f3afb10..3f9837f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3502,6 +3502,52 @@ long_mul(PyLongObject *a, PyLongObject *b) return (PyObject *)z; } +/* Fast modulo division for single-digit longs. */ +static PyObject * +fast_mod(PyLongObject *a, PyLongObject *b) +{ + sdigit left = a->ob_digit[0]; + sdigit right = b->ob_digit[0]; + sdigit mod; + + assert(Py_ABS(Py_SIZE(a)) == 1); + assert(Py_ABS(Py_SIZE(b)) == 1); + + if (Py_SIZE(a) == Py_SIZE(b)) { + /* 'a' and 'b' have the same sign. */ + mod = left % right; + } + else { + /* Either 'a' or 'b' is negative. */ + mod = right - 1 - (left - 1) % right; + } + + return PyLong_FromLong(mod * Py_SIZE(b)); +} + +/* Fast floor division for single-digit longs. */ +static PyObject * +fast_floor_div(PyLongObject *a, PyLongObject *b) +{ + sdigit left = a->ob_digit[0]; + sdigit right = b->ob_digit[0]; + sdigit div; + + assert(Py_ABS(Py_SIZE(a)) == 1); + assert(Py_ABS(Py_SIZE(b)) == 1); + + if (Py_SIZE(a) == Py_SIZE(b)) { + /* 'a' and 'b' have the same sign. */ + div = left / right; + } + else { + /* Either 'a' or 'b' is negative. */ + div = -1 - (left - 1) / right; + } + + return PyLong_FromLong(div); +} + /* The / and % operators are now defined in terms of divmod(). The expression a mod b has the value a - b*floor(a/b). The long_divrem function gives the remainder after division of @@ -3529,6 +3575,30 @@ l_divmod(PyLongObject *v, PyLongObject *w, { PyLongObject *div, *mod; + if (Py_ABS(Py_SIZE(v)) == 1 && Py_ABS(Py_SIZE(w)) == 1) { + /* Fast path for single-digit longs */ + div = NULL; + if (pdiv != NULL) { + div = (PyLongObject *)fast_floor_div(v, w); + if (div == NULL) { + return -1; + } + } + if (pmod != NULL) { + mod = (PyLongObject *)fast_mod(v, w); + if (mod == NULL) { + Py_XDECREF(div); + return -1; + } + *pmod = mod; + } + if (pdiv != NULL) { + /* We only want to set `*pdiv` when `*pmod` is + set successfully. */ + *pdiv = div; + } + return 0; + } if (long_divrem(v, w, &div, &mod) < 0) return -1; if ((Py_SIZE(mod) < 0 && Py_SIZE(w) > 0) || @@ -3573,6 +3643,11 @@ long_div(PyObject *a, PyObject *b) PyLongObject *div; CHECK_BINOP(a, b); + + if (Py_ABS(Py_SIZE(a)) == 1 && Py_ABS(Py_SIZE(b)) == 1) { + return fast_floor_div((PyLongObject*)a, (PyLongObject*)b); + } + if (l_divmod((PyLongObject*)a, (PyLongObject*)b, &div, NULL) < 0) div = NULL; return (PyObject *)div; @@ -3848,6 +3923,10 @@ long_mod(PyObject *a, PyObject *b) CHECK_BINOP(a, b); + if (Py_ABS(Py_SIZE(a)) == 1 && Py_ABS(Py_SIZE(b)) == 1) { + return fast_mod((PyLongObject*)a, (PyLongObject*)b); + } + if (l_divmod((PyLongObject*)a, (PyLongObject*)b, NULL, &mod) < 0) mod = NULL; return (PyObject *)mod; -- cgit v0.12 From bc1ee460dcbe5811ebb10a6cacd8d3670846f039 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sat, 13 Feb 2016 00:41:37 +0000 Subject: Issue #25179: Documentation for formatted string literals aka f-strings Some of the inspiration and wording is taken from the text of PEP 498 by Eric V. Smith, and the existing str.format() documentation. --- Doc/faq/programming.rst | 3 +- Doc/library/datetime.rst | 9 ++-- Doc/library/enum.rst | 3 +- Doc/library/stdtypes.rst | 15 +++--- Doc/library/string.rst | 10 ++-- Doc/reference/datamodel.rst | 5 +- Doc/reference/lexical_analysis.rst | 108 ++++++++++++++++++++++++++++++++++++- Doc/tutorial/inputoutput.rst | 3 +- Doc/tutorial/introduction.rst | 3 ++ Doc/whatsnew/3.6.rst | 20 ++++++- 10 files changed, 158 insertions(+), 21 deletions(-) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9fba9fe..7b529a1 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -839,7 +839,8 @@ How do I convert a number to a string? To convert, e.g., the number 144 to the string '144', use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`formatstrings` section, e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections, +e.g. ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1f3edd2..a822842 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -605,7 +605,8 @@ Instance methods: .. method:: date.__format__(format) Same as :meth:`.date.strftime`. This makes it possible to specify a format - string for a :class:`.date` object when using :meth:`str.format`. For a + string for a :class:`.date` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1180,7 +1181,8 @@ Instance methods: .. method:: datetime.__format__(format) Same as :meth:`.datetime.strftime`. This makes it possible to specify a format - string for a :class:`.datetime` object when using :meth:`str.format`. For a + string for a :class:`.datetime` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. @@ -1425,7 +1427,8 @@ Instance methods: .. method:: time.__format__(format) Same as :meth:`.time.strftime`. This makes it possible to specify a format string - for a :class:`.time` object when using :meth:`str.format`. For a + for a :class:`.time` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. For a complete list of formatting directives, see :ref:`strftime-strptime-behavior`. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 333adfc..b3691ca 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -558,7 +558,8 @@ Some rules: 4. %-style formatting: `%s` and `%r` call the :class:`Enum` class's :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :meth:`str.format` (or :func:`format`) will use the mixed-in +5. :ref:`Formatted string literals `, :meth:`str.format`, + and :func:`format` will use the mixed-in type's :meth:`__format__`. If the :class:`Enum` class's :func:`str` or :func:`repr` is desired, use the `!s` or `!r` format codes. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index e491fd2..4aebcb8 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1450,8 +1450,8 @@ multiple fragments. For more information on the ``str`` class and its methods, see :ref:`textseq` and the :ref:`string-methods` section below. To output - formatted strings, see the :ref:`formatstrings` section. In addition, - see the :ref:`stringservices` section. + formatted strings, see the :ref:`f-strings` and :ref:`formatstrings` + sections. In addition, see the :ref:`stringservices` section. .. index:: @@ -2053,8 +2053,8 @@ expression support in the :mod:`re` module). .. index:: single: formatting, string (%) single: interpolation, string (%) - single: string; formatting - single: string; interpolation + single: string; formatting, printf + single: string; interpolation, printf single: printf-style formatting single: sprintf-style formatting single: % formatting @@ -2064,9 +2064,10 @@ expression support in the :mod:`re` module). The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :meth:`str.format` interface - helps avoid these errors, and also provides a generally more powerful, - flexible and extensible approach to formatting text. + dictionaries correctly). Using the newer :ref:`formatted + string literals ` or the :meth:`str.format` interface + helps avoid these errors. These alternatives also provide more powerful, + flexible and extensible approaches to formatting text. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5b917d9..cda8e86 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -188,7 +188,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, -subclasses can define their own format string syntax). +subclasses can define their own format string syntax). The syntax is +related to that of :ref:`formatted string literals `, but +there are differences. Format strings contain "replacement fields" surrounded by curly braces ``{}``. Anything that is not contained in braces is considered literal text, which is @@ -283,7 +285,8 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings`). They can also be passed directly to the built-in +:ref:`formatstrings` and :ref:`f-strings`). +They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -308,7 +311,8 @@ The general form of a *standard format specifier* is: If a valid *align* value is specified, it can be preceded by a *fill* character that can be any character and defaults to a space if omitted. It is not possible to use a literal curly brace ("``{``" or "``}``") as -the *fill* character when using the :meth:`str.format` +the *fill* character in a :ref:`formatted string literal +` or when using the :meth:`str.format` method. However, it is possible to insert a curly brace with a nested replacement field. This limitation doesn't affect the :func:`format` function. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 764c491..56ec0d0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1234,8 +1234,9 @@ Basic customization .. method:: object.__format__(self, format_spec) - Called by the :func:`format` built-in function (and by extension, the - :meth:`str.format` method of class :class:`str`) to produce a "formatted" + Called by the :func:`format` built-in function, + and by extension, evaluation of :ref:`formatted string literals + ` and the :meth:`str.format` method, to produce a "formatted" string representation of an object. The ``format_spec`` argument is a string that contains a description of the formatting options desired. The interpretation of the ``format_spec`` argument is up to the type diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 8ad975f..aad5b0c 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -405,7 +405,8 @@ String literals are described by the following lexical definitions: .. productionlist:: stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" + stringprefix: "r" | "u" | "R" | "U" | "f" | "F" + : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' shortstringitem: `shortstringchar` | `stringescapeseq` @@ -464,6 +465,11 @@ is not supported. to simplify the maintenance of dual Python 2.x and 3.x codebases. See :pep:`414` for more information. +A string literal with ``'f'`` or ``'F'`` in its prefix is a +:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be +combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw +formatted strings are possible, but formatted bytes literals are not. + In triple-quoted literals, unescaped newlines and quotes are allowed (and are retained), except that three unescaped quotes in a row terminate the literal. (A "quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) @@ -584,7 +590,105 @@ comments to parts of strings, for example:: Note that this feature is defined at the syntactical level, but implemented at compile time. The '+' operator must be used to concatenate string expressions at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings). +styles for each component (even mixing raw strings and triple quoted strings), +and formatted string literals may be concatenated with plain string literals. + + +.. index:: + single: formatted string literal + single: interpolated string literal + single: string; formatted literal + single: string; interpolated literal + single: f-string +.. _f-strings: + +Formatted string literals +------------------------- + +.. versionadded:: 3.6 + +A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal +that is prefixed with ``'f'`` or ``'F'``. These strings may contain +replacement fields, which are expressions delimited by curly braces ``{}``. +While other string literals always have a constant value, formatted strings +are really expressions evaluated at run time. + +Escape sequences are decoded like in ordinary string literals (except when +a literal is also marked as a raw string). After decoding, the grammar +for the contents of the string is: + +.. productionlist:: + f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* + replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" + f_expression: `conditional_expression` ("," `conditional_expression`)* [","] + : | `yield_expression` + conversion: "s" | "r" | "a" + format_spec: (`literal_char` | NULL | `replacement_field`)* + literal_char: + +The parts of the string outside curly braces are treated literally, +except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced +with the corresponding single curly brace. A single opening curly +bracket ``'{'`` marks a replacement field, which starts with a +Python expression. After the expression, there may be a conversion field, +introduced by an exclamation point ``'!'``. A format specifier may also +be appended, introduced by a colon ``':'``. A replacement field ends +with a closing curly bracket ``'}'``. + +Expressions in formatted string literals are treated like regular +Python expressions surrounded by parentheses, with a few exceptions. +An empty expression is not allowed, and a :keyword:`lambda` expression +must be surrounded by explicit parentheses. Replacement expressions +can contain line breaks (e.g. in triple-quoted strings), but they +cannot contain comments. Each expression is evaluated in the context +where the formatted string literal appears, in order from left to right. + +If a conversion is specified, the result of evaluating the expression +is converted before formatting. Conversion ``'!s'`` calls :func:`str` on +the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. + +The result is then formatted using the :func:`format` protocol. The +format specifier is passed to the :meth:`__format__` method of the +expression or conversion result. An empty string is passed when the +format specifier is omitted. The formatted result is then included in +the final value of the whole string. + +Top-level format specifiers may include nested replacement fields. +These nested fields may include their own conversion fields and +format specifiers, but may not include more deeply-nested replacement fields. + +Formatted string literals may be concatenated, but replacement fields +cannot be split across literals. + +Some examples of formatted string literals:: + + >>> name = "Fred" + >>> f"He said his name is {name!r}." + "He said his name is 'Fred'." + >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r + "He said his name is 'Fred'." + >>> width = 10 + >>> precision = 4 + >>> value = decimal.Decimal("12.34567") + >>> f"result: {value:{width}.{precision}}" # nested fields + 'result: 12.35' + +A consequence of sharing the same syntax as regular string literals is +that characters in the replacement fields must not conflict with the +quoting used in the outer formatted string literal. Also, escape +sequences normally apply to the outer formatted string literal, +rather than inner string literals:: + + f"abc {a["x"]} def" # error: outer string literal ended prematurely + f"abc {a[\"x\"]} def" # workaround: escape the inner quotes + f"abc {a['x']} def" # workaround: use different quoting + + f"newline: {ord('\n')}" # error: literal line break in inner string + f"newline: {ord('\\n')}" # workaround: double escaping + fr"newline: {ord('\n')}" # workaround: raw outer string + +See also :pep:`498` for the proposal that added formatted string literals, +and :meth:`str.format`, which uses a related format string mechanism. .. _numbers: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 5314fed..f2171f4 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -25,7 +25,8 @@ first way is to do all the string handling yourself; using string slicing and concatenation operations you can create any layout you can imagine. The string type has some methods that perform useful operations for padding strings to a given column width; these will be discussed shortly. The second -way is to use the :meth:`str.format` method. +way is to use :ref:`formatted string literals `, or the +:meth:`str.format` method. The :mod:`string` module contains a :class:`~string.Template` class which offers yet another way to substitute values into strings. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 8758f38..87f0fa5 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -352,6 +352,9 @@ The built-in function :func:`len` returns the length of a string:: Strings support a large number of methods for basic transformations and searching. + :ref:`f-strings` + String literals that have embedded expressions. + :ref:`formatstrings` Information about string formatting with :meth:`str.format`. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b7d14f2..8b879de 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -62,7 +62,7 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.6. Brevity is key. -* None yet. +* PEP 498: :ref:`Formatted string literals ` .. PEP-sized items next. @@ -80,6 +80,24 @@ Summary -- Release highlights PEP written by Carl Meyer +.. _whatsnew-fstrings: + +PEP 498: Formatted string literals +---------------------------------- + +Formatted string literals are a new kind of string literal, prefixed +with ``'f'``. They are similar to the format strings accepted by +:meth:`str.format`. They contain replacement fields surrounded by +curly braces. The replacement fields are expressions, which are +evaluated at run time, and then formatted using the :func:`format` protocol. + + >>> name = "Fred" + >>> f"He said his name is {name}." + 'He said his name is Fred.' + +See :pep:`498` and the main documentation at :ref:`f-strings`. + + Other Language Changes ====================== -- cgit v0.12 From b6f17f5210843295ae87c779596acbe6835b36f0 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 14 Feb 2016 01:41:35 -0800 Subject: The return type of a rich comparison is an int --- Modules/_bisectmodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 02b55d1..22ddbf2 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -12,7 +12,8 @@ static Py_ssize_t internal_bisect_right(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi) { PyObject *litem; - Py_ssize_t mid, res; + Py_ssize_t mid; + int res; if (lo < 0) { PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); @@ -115,7 +116,8 @@ static Py_ssize_t internal_bisect_left(PyObject *list, PyObject *item, Py_ssize_t lo, Py_ssize_t hi) { PyObject *litem; - Py_ssize_t mid, res; + Py_ssize_t mid; + int res; if (lo < 0) { PyErr_SetString(PyExc_ValueError, "lo must be non-negative"); -- cgit v0.12 From 40383c8f10c6a34aee6f513dc30b2005951cc30b Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 15 Feb 2016 17:50:33 +0100 Subject: Minor clarification in tutorial. --- Doc/tutorial/controlflow.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 813c828..8453796 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -78,6 +78,9 @@ slice notation makes this especially convenient:: >>> words ['defenestrate', 'cat', 'window', 'defenestrate'] +With ``for w in words:``, the example would attempt to create an infinite list, +inserting ``defenestrate`` over and over again. + .. _tut-range: -- cgit v0.12 From 4cbab346dfcea1f221bfb0811fabed8dfb435e7e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 20 Feb 2016 18:45:56 -0800 Subject: Issue #26397: Update an importlib example to use util.module_from_spec() instead of create_module() --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 4d429bb..17a65aa 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1389,7 +1389,7 @@ Python 3.6 and newer for other parts of the code). break else: raise ImportError(f'No module named {absolute_name!r}') - module = spec.loader.create_module(spec) + module = util.module_from_spec(spec) spec.loader.exec_module(module) sys.modules[absolute_name] = module if path is not None: -- cgit v0.12 From 86a8be00ed5266550a3d97a3c065f7a22018b6a6 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 20 Feb 2016 18:47:09 -0800 Subject: Fix a name in an example --- Doc/library/importlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 17a65aa..2bb586c3 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1389,7 +1389,7 @@ Python 3.6 and newer for other parts of the code). break else: raise ImportError(f'No module named {absolute_name!r}') - module = util.module_from_spec(spec) + module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) sys.modules[absolute_name] = module if path is not None: -- cgit v0.12 From d9108d1253eef9f43c60e0ed5f1ec5603c1fba6c Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 21 Feb 2016 08:49:56 +0000 Subject: Issue #23430: Stop socketserver from catching SystemExit etc from handlers Also make handle_error() consistently output to stderr, and fix the documentation. --- Doc/library/socketserver.rst | 6 ++- Doc/whatsnew/3.6.rst | 11 +++++- Lib/socketserver.py | 31 +++++++++------ Lib/test/test_socketserver.py | 92 +++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 6 +++ 5 files changed, 131 insertions(+), 15 deletions(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index f0cfc80..aaaa61e 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -304,7 +304,11 @@ Server Objects This function is called if the :meth:`~BaseRequestHandler.handle` method of a :attr:`RequestHandlerClass` instance raises an exception. The default action is to print the traceback to - standard output and continue handling further requests. + standard error and continue handling further requests. + + .. versionchanged:: 3.6 + Now only called for exceptions derived from the :exc:`Exception` + class. .. method:: handle_timeout() diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8b879de..5c02b7d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -334,7 +334,16 @@ Changes in the Python API * When a relative import is performed and no parent package is known, then :exc:`ImportError` will be raised. Previously, :exc:`SystemError` could be - raised. (Contribute by Brett Cannon in :issue:`18018`.) + raised. (Contributed by Brett Cannon in :issue:`18018`.) + +* Servers based on the :mod:`socketserver` module, including those + defined in :mod:`http.server`, :mod:`xmlrpc.server` and + :mod:`wsgiref.simple_server`, now only catch exceptions derived + from :exc:`Exception`. Therefore if a request handler raises + an exception like :exc:`SystemExit` or :exc:`KeyboardInterrupt`, + :meth:`~socketserver.BaseServer.handle_error` is no longer called, and + the exception will stop a single-threaded server. (Contributed by + Martin Panter in :issue:`23430`.) Changes in the C API diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 1524d16..64689991 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -132,6 +132,7 @@ import socket import selectors import os import errno +import sys try: import threading except ImportError: @@ -316,9 +317,12 @@ class BaseServer: if self.verify_request(request, client_address): try: self.process_request(request, client_address) - except: + except Exception: self.handle_error(request, client_address) self.shutdown_request(request) + except: + self.shutdown_request(request) + raise else: self.shutdown_request(request) @@ -372,12 +376,12 @@ class BaseServer: The default is to print a traceback and continue. """ - print('-'*40) - print('Exception happened during processing of request from', end=' ') - print(client_address) + print('-'*40, file=sys.stderr) + print('Exception happened during processing of request from', + client_address, file=sys.stderr) import traceback - traceback.print_exc() # XXX But this goes to stderr! - print('-'*40) + traceback.print_exc() + print('-'*40, file=sys.stderr) class TCPServer(BaseServer): @@ -601,16 +605,17 @@ class ForkingMixIn: else: # Child process. # This must never return, hence os._exit()! + status = 1 try: self.finish_request(request, client_address) - self.shutdown_request(request) - os._exit(0) - except: + status = 0 + except Exception: + self.handle_error(request, client_address) + finally: try: - self.handle_error(request, client_address) self.shutdown_request(request) finally: - os._exit(1) + os._exit(status) class ThreadingMixIn: @@ -628,9 +633,9 @@ class ThreadingMixIn: """ try: self.finish_request(request, client_address) - self.shutdown_request(request) - except: + except Exception: self.handle_error(request, client_address) + finally: self.shutdown_request(request) def process_request(self, request, client_address): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index dc23210..bff0ff4 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -58,6 +58,7 @@ if HAVE_UNIX_SOCKETS: @contextlib.contextmanager def simple_subprocess(testcase): + """Tests that a custom child process is not waited on (Issue 1540386)""" pid = os.fork() if pid == 0: # Don't raise an exception; it would be caught by the test harness. @@ -281,6 +282,97 @@ class SocketServerTest(unittest.TestCase): socketserver.StreamRequestHandler) +class ErrorHandlerTest(unittest.TestCase): + """Test that the servers pass normal exceptions from the handler to + handle_error(), and that exiting exceptions like SystemExit and + KeyboardInterrupt are not passed.""" + + def tearDown(self): + test.support.unlink(test.support.TESTFN) + + def test_sync_handled(self): + BaseErrorTestServer(ValueError) + self.check_result(handled=True) + + def test_sync_not_handled(self): + with self.assertRaises(SystemExit): + BaseErrorTestServer(SystemExit) + self.check_result(handled=False) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_handled(self): + ThreadingErrorTestServer(ValueError) + self.check_result(handled=True) + + @unittest.skipUnless(threading, 'Threading required for this test.') + def test_threading_not_handled(self): + ThreadingErrorTestServer(SystemExit) + self.check_result(handled=False) + + @requires_forking + def test_forking_handled(self): + ForkingErrorTestServer(ValueError) + self.check_result(handled=True) + + @requires_forking + def test_forking_not_handled(self): + ForkingErrorTestServer(SystemExit) + self.check_result(handled=False) + + def check_result(self, handled): + with open(test.support.TESTFN) as log: + expected = 'Handler called\n' + 'Error handled\n' * handled + self.assertEqual(log.read(), expected) + + +class BaseErrorTestServer(socketserver.TCPServer): + def __init__(self, exception): + self.exception = exception + super().__init__((HOST, 0), BadHandler) + with socket.create_connection(self.server_address): + pass + try: + self.handle_request() + finally: + self.server_close() + self.wait_done() + + def handle_error(self, request, client_address): + with open(test.support.TESTFN, 'a') as log: + log.write('Error handled\n') + + def wait_done(self): + pass + + +class BadHandler(socketserver.BaseRequestHandler): + def handle(self): + with open(test.support.TESTFN, 'a') as log: + log.write('Handler called\n') + raise self.server.exception('Test error') + + +class ThreadingErrorTestServer(socketserver.ThreadingMixIn, + BaseErrorTestServer): + def __init__(self, *pos, **kw): + self.done = threading.Event() + super().__init__(*pos, **kw) + + def shutdown_request(self, *pos, **kw): + super().shutdown_request(*pos, **kw) + self.done.set() + + def wait_done(self): + self.done.wait() + + +class ForkingErrorTestServer(socketserver.ForkingMixIn, BaseErrorTestServer): + def wait_done(self): + [child] = self.active_children + os.waitpid(child, 0) + self.active_children.clear() + + class MiscTestCase(unittest.TestCase): def test_all(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6c1e83f..4043e01 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -196,6 +196,12 @@ Issue #26186: Remove an invalid type check in importlib.util.LazyLoader. the connected socket) when verify_request() returns false. Patch by Aviv Palivoda. +- Issue #23430: Change the socketserver module to only catch exceptions + raised from a request handler that are derived from Exception (instead of + BaseException). Therefore SystemExit and KeyboardInterrupt no longer + trigger the handle_error() method, and will now to stop a single-threaded + server. + - Issue #25939: On Windows open the cert store readonly in ssl.enum_certificates. - Issue #25995: os.walk() no longer uses FDs proportional to the tree depth. -- cgit v0.12 From 62bc32181901061fbadfc7e68c518871f6d8faea Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Thu, 25 Feb 2016 00:25:45 -0800 Subject: Fix out-of-date comment --- Lib/collections/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 9a26a93..e5497b3 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -842,7 +842,7 @@ class Counter(dict): ######################################################################## -### ChainMap (helper for configparser and string.Template) +### ChainMap ######################################################################## class ChainMap(MutableMapping): -- cgit v0.12 From 202803a0c00f00e62ad613fff119d7a5432f342c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 22:34:45 -0800 Subject: c_encoding can never be NULL, which allows some code simplification --- Python/ast.c | 90 +++++++++++++++++++++++++++++------------------------------- 1 file changed, 43 insertions(+), 47 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index d19546a..c342609 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -4002,55 +4002,52 @@ decode_unicode(struct compiling *c, const char *s, size_t len, const char *encod char *p; const char *end; - if (encoding == NULL) { - u = NULL; - } else { - /* check for integer overflow */ - if (len > PY_SIZE_MAX / 6) - return NULL; - /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 - "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ - u = PyBytes_FromStringAndSize((char *)NULL, len * 6); - if (u == NULL) - return NULL; - p = buf = PyBytes_AsString(u); - end = s + len; - while (s < end) { - if (*s == '\\') { - *p++ = *s++; - if (*s & 0x80) { - strcpy(p, "u005c"); - p += 5; - } + /* check for integer overflow */ + if (len > PY_SIZE_MAX / 6) + return NULL; + /* "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 + "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), or ~1:6 */ + u = PyBytes_FromStringAndSize((char *)NULL, len * 6); + if (u == NULL) + return NULL; + p = buf = PyBytes_AsString(u); + end = s + len; + while (s < end) { + if (*s == '\\') { + *p++ = *s++; + if (*s & 0x80) { + strcpy(p, "u005c"); + p += 5; } - if (*s & 0x80) { /* XXX inefficient */ - PyObject *w; - int kind; - void *data; - Py_ssize_t len, i; - w = decode_utf8(c, &s, end); - if (w == NULL) { - Py_DECREF(u); - return NULL; - } - kind = PyUnicode_KIND(w); - data = PyUnicode_DATA(w); - len = PyUnicode_GET_LENGTH(w); - for (i = 0; i < len; i++) { - Py_UCS4 chr = PyUnicode_READ(kind, data, i); - sprintf(p, "\\U%08x", chr); - p += 10; - } - /* Should be impossible to overflow */ - assert(p - buf <= Py_SIZE(u)); - Py_DECREF(w); - } else { - *p++ = *s++; + } + if (*s & 0x80) { /* XXX inefficient */ + PyObject *w; + int kind; + void *data; + Py_ssize_t len, i; + w = decode_utf8(c, &s, end); + if (w == NULL) { + Py_DECREF(u); + return NULL; } + kind = PyUnicode_KIND(w); + data = PyUnicode_DATA(w); + len = PyUnicode_GET_LENGTH(w); + for (i = 0; i < len; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + sprintf(p, "\\U%08x", chr); + p += 10; + } + /* Should be impossible to overflow */ + assert(p - buf <= Py_SIZE(u)); + Py_DECREF(w); + } else { + *p++ = *s++; } - len = p - buf; - s = buf; } + len = p - buf; + s = buf; + v = PyUnicode_DecodeUnicodeEscape(s, len, NULL); Py_XDECREF(u); return v; @@ -4994,8 +4991,7 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) } } } - need_encoding = (!*bytesmode && c->c_encoding != NULL && - strcmp(c->c_encoding, "utf-8") != 0); + need_encoding = !*bytesmode && strcmp(c->c_encoding, "utf-8") != 0; if (rawmode || strchr(s, '\\') == NULL) { if (need_encoding) { PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); -- cgit v0.12 From 768921cf3360822dc252d7e295e94960afa9feee Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 23:13:53 -0800 Subject: rewrite parsestr() so it's comprehensible; remove dead code --- Python/ast.c | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index c342609..ba6b4ba 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -3995,7 +3995,7 @@ decode_utf8(struct compiling *c, const char **sPtr, const char *end) } static PyObject * -decode_unicode(struct compiling *c, const char *s, size_t len, const char *encoding) +decode_unicode_with_escapes(struct compiling *c, const char *s, size_t len) { PyObject *v, *u; char *buf; @@ -4921,7 +4921,6 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) const char *s = STR(n); int quote = Py_CHARMASK(*s); int rawmode = 0; - int need_encoding; if (Py_ISALPHA(quote)) { while (!*bytesmode || !rawmode) { if (quote == 'b' || quote == 'B') { @@ -4977,11 +4976,10 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) return NULL; } } - if (!*bytesmode && !rawmode) { - return decode_unicode(c, s, len, c->c_encoding); - } + /* Avoid invoking escape decoding routines if possible. */ + rawmode = rawmode || strchr(s, '\\') == NULL; if (*bytesmode) { - /* Disallow non-ascii characters (but not escapes) */ + /* Disallow non-ASCII characters. */ const char *ch; for (ch = s; *ch; ch++) { if (Py_CHARMASK(*ch) >= 0x80) { @@ -4990,26 +4988,16 @@ parsestr(struct compiling *c, const node *n, int *bytesmode, int *fmode) return NULL; } } - } - need_encoding = !*bytesmode && strcmp(c->c_encoding, "utf-8") != 0; - if (rawmode || strchr(s, '\\') == NULL) { - if (need_encoding) { - PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL); - if (u == NULL || !*bytesmode) - return u; - v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL); - Py_DECREF(u); - return v; - } else if (*bytesmode) { + if (rawmode) return PyBytes_FromStringAndSize(s, len); - } else if (strcmp(c->c_encoding, "utf-8") == 0) { - return PyUnicode_FromStringAndSize(s, len); - } else { - return PyUnicode_DecodeLatin1(s, len, NULL); - } + else + return PyBytes_DecodeEscape(s, len, NULL, /* ignored */ 0, NULL); + } else { + if (rawmode) + return PyUnicode_DecodeUTF8Stateful(s, len, NULL, NULL); + else + return decode_unicode_with_escapes(c, s, len); } - return PyBytes_DecodeEscape(s, len, NULL, 1, - need_encoding ? c->c_encoding : NULL); } /* Accepts a STRING+ atom, and produces an expr_ty node. Run through -- cgit v0.12 From 9d66d4af06ad15226b271988cd7f5d0c681924af Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 25 Feb 2016 23:25:14 -0800 Subject: remove unused c_encoding struct member --- Python/ast.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/Python/ast.c b/Python/ast.c index ba6b4ba..ad771cf 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -574,7 +574,6 @@ PyAST_Validate(mod_ty mod) /* Data structure used internally */ struct compiling { - char *c_encoding; /* source encoding */ PyArena *c_arena; /* Arena for allocating memory. */ PyObject *c_filename; /* filename */ PyObject *c_normalize; /* Normalization function from unicodedata. */ @@ -761,23 +760,11 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, c.c_arena = arena; /* borrowed reference */ c.c_filename = filename; - c.c_normalize = c.c_normalize_args = NULL; - if (flags && flags->cf_flags & PyCF_SOURCE_IS_UTF8) { - c.c_encoding = "utf-8"; - if (TYPE(n) == encoding_decl) { -#if 0 - ast_error(c, n, "encoding declaration in Unicode string"); - goto out; -#endif - n = CHILD(n, 0); - } - } else if (TYPE(n) == encoding_decl) { - c.c_encoding = STR(n); + c.c_normalize = NULL; + c.c_normalize_args = NULL; + + if (TYPE(n) == encoding_decl) n = CHILD(n, 0); - } else { - /* PEP 3120 */ - c.c_encoding = "utf-8"; - } k = 0; switch (TYPE(n)) { -- cgit v0.12 From fc6f2efd13a6b667e48cdf1f5cf68d34fcb0fa74 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 27 Feb 2016 02:19:22 +0100 Subject: compile.c: inline compiler_use_new_block() * Inline compiler_use_new_block() function into its only callee, compiler_enter_scope() * Remove unused NEW_BLOCK() macro --- Python/compile.c | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index ca1d865..568bdfe 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -171,7 +171,6 @@ static int compiler_addop(struct compiler *, int); static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *); static int compiler_addop_i(struct compiler *, int, Py_ssize_t); static int compiler_addop_j(struct compiler *, int, basicblock *, int); -static basicblock *compiler_use_new_block(struct compiler *); static int compiler_error(struct compiler *, const char *); static int compiler_nameop(struct compiler *, identifier, expr_context_ty); @@ -523,6 +522,7 @@ compiler_enter_scope(struct compiler *c, identifier name, int scope_type, void *key, int lineno) { struct compiler_unit *u; + basicblock *block; u = (struct compiler_unit *)PyObject_Malloc(sizeof( struct compiler_unit)); @@ -620,8 +620,11 @@ compiler_enter_scope(struct compiler *c, identifier name, c->u = u; c->c_nestlevel++; - if (compiler_use_new_block(c) == NULL) + + block = compiler_new_block(c); + if (block == NULL) return 0; + c->u->u_curblock = block; if (u->u_scope_type != COMPILER_SCOPE_MODULE) { if (!compiler_set_qualname(c)) @@ -756,16 +759,6 @@ compiler_new_block(struct compiler *c) } static basicblock * -compiler_use_new_block(struct compiler *c) -{ - basicblock *block = compiler_new_block(c); - if (block == NULL) - return NULL; - c->u->u_curblock = block; - return block; -} - -static basicblock * compiler_next_block(struct compiler *c) { basicblock *block = compiler_new_block(c); @@ -1208,22 +1201,12 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute) return 1; } -/* The distinction between NEW_BLOCK and NEXT_BLOCK is subtle. (I'd - like to find better names.) NEW_BLOCK() creates a new block and sets - it as the current block. NEXT_BLOCK() also creates an implicit jump - from the current block to the new block. -*/ +/* NEXT_BLOCK() creates an implicit jump from the current block + to the new block. -/* The returns inside these macros make it impossible to decref objects - created in the local function. Local objects should use the arena. + The returns inside this macro make it impossible to decref objects + created in the local function. Local objects should use the arena. */ - - -#define NEW_BLOCK(C) { \ - if (compiler_use_new_block((C)) == NULL) \ - return 0; \ -} - #define NEXT_BLOCK(C) { \ if (compiler_next_block((C)) == NULL) \ return 0; \ -- cgit v0.12 From 2ad474ba5e8ac06b7edae3cd2e15ab93212dd01b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 1 Mar 2016 23:34:47 +0100 Subject: Update assertion in compiler_addop_i() In practice, bytecode instruction arguments are unsigned. Update the assertion to make it more explicit that argument must be greater or equal than 0. Rewrite also the comment. --- Python/compile.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 568bdfe..e86b293 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1163,10 +1163,14 @@ compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg) struct instr *i; int off; - /* Integer arguments are limit to 16-bit. There is an extension for 32-bit - integer arguments. */ - assert((-2147483647-1) <= oparg); - assert(oparg <= 2147483647); + /* oparg value is unsigned, but a signed C int is usually used to store + it in the C code (like Python/ceval.c). + + Limit to 32-bit signed C int (rather than INT_MAX) for portability. + + The argument of a concrete bytecode instruction is limited to 16-bit. + EXTENDED_ARG is used for 32-bit arguments. */ + assert(0 <= oparg && oparg <= 2147483647); off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) -- cgit v0.12 From 589106b20683b95aac1fe29cf9db5aed632f1cf3 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 2 Mar 2016 00:06:21 -0800 Subject: Put block length computations in a more logical order. --- Modules/_collectionsmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 0b7a88f..309dfd2 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -589,8 +589,8 @@ deque_clear(dequeobject *deque) /* Now the old size, leftblock, and leftindex are disconnected from the empty deque and we can use them to decref the pointers. */ - itemptr = &leftblock->data[leftindex]; m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; + itemptr = &leftblock->data[leftindex]; limit = &leftblock->data[leftindex + m]; n -= m; while (1) { @@ -600,8 +600,8 @@ deque_clear(dequeobject *deque) CHECK_NOT_END(leftblock->rightlink); prevblock = leftblock; leftblock = leftblock->rightlink; - itemptr = leftblock->data; m = (n > BLOCKLEN) ? BLOCKLEN : n; + itemptr = leftblock->data; limit = &leftblock->data[m]; n -= m; freeblock(prevblock); -- cgit v0.12 From 6f86a3308a089e80be2562316dec142d65c85451 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 2 Mar 2016 00:30:58 -0800 Subject: Factor-out common subexpression. --- Modules/_collectionsmodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 309dfd2..7dcd7e7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -572,9 +572,9 @@ deque_clear(dequeobject *deque) } /* Remember the old size, leftblock, and leftindex */ + n = Py_SIZE(deque); leftblock = deque->leftblock; leftindex = deque->leftindex; - n = Py_SIZE(deque); /* Set the deque to be empty using the newly allocated block */ MARK_END(b->leftlink); @@ -591,7 +591,7 @@ deque_clear(dequeobject *deque) */ m = (BLOCKLEN - leftindex > n) ? n : BLOCKLEN - leftindex; itemptr = &leftblock->data[leftindex]; - limit = &leftblock->data[leftindex + m]; + limit = itemptr + m; n -= m; while (1) { if (itemptr == limit) { @@ -602,7 +602,7 @@ deque_clear(dequeobject *deque) leftblock = leftblock->rightlink; m = (n > BLOCKLEN) ? BLOCKLEN : n; itemptr = leftblock->data; - limit = &leftblock->data[m]; + limit = itemptr + m; n -= m; freeblock(prevblock); } -- cgit v0.12 From 3ebaea005dc6ab4a06fb585c23321b9bb0398b90 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Mar 2016 10:43:45 -0500 Subject: Sync selectors.py with upstream asyncio --- Lib/selectors.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/selectors.py b/Lib/selectors.py index ecd8632..d8769e3 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -49,11 +49,12 @@ SelectorKey.__doc__ = """SelectorKey(fileobj, fd, events, data) Object used to associate a file object to its backing file descriptor, selected event mask, and attached data. """ -SelectorKey.fileobj.__doc__ = 'File object registered.' -SelectorKey.fd.__doc__ = 'Underlying file descriptor.' -SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' -SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. -For example, this could be used to store a per-client session ID.''') +if sys.version_info >= (3, 5): + SelectorKey.fileobj.__doc__ = 'File object registered.' + SelectorKey.fd.__doc__ = 'Underlying file descriptor.' + SelectorKey.events.__doc__ = 'Events that must be waited for on this file object.' + SelectorKey.data.__doc__ = ('''Optional opaque data associated to this file object. + For example, this could be used to store a per-client session ID.''') class _SelectorMapping(Mapping): """Mapping of file objects to selector keys.""" -- cgit v0.12 From 6282e656e994a8722f39db6aea99aad2a826dee6 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Wed, 2 Mar 2016 19:30:18 +0200 Subject: Issue #26335: Make mmap.write() return the number of bytes written like other write methods. Patch by Jakub Stasiak. --- Doc/library/mmap.rst | 9 +++++++-- Lib/test/test_mmap.py | 8 ++++++++ Misc/NEWS | 3 +++ Modules/mmapmodule.c | 5 +++-- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 33baf2b..fb24728 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -263,13 +263,18 @@ To map anonymous memory, -1 should be passed as the fileno along with the length .. method:: write(bytes) Write the bytes in *bytes* into memory at the current position of the - file pointer; the file position is updated to point after the bytes that - were written. If the mmap was created with :const:`ACCESS_READ`, then + file pointer and return the number of bytes written (never less than + ``len(bytes)``, since if the write fails, a :exc:`ValueError` will be + raised). The file position is updated to point after the bytes that + were written. If the mmap was created with :const:`ACCESS_READ`, then writing to it will raise a :exc:`TypeError` exception. .. versionchanged:: 3.5 Writable :term:`bytes-like object` is now accepted. + .. versionchanged:: 3.6 + The number of bytes written is now returned. + .. method:: write_byte(byte) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0f25742..be2e65a 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -713,6 +713,14 @@ class MmapTests(unittest.TestCase): gc_collect() self.assertIs(wr(), None) + def test_write_returning_the_number_of_bytes_written(self): + mm = mmap.mmap(-1, 16) + self.assertEqual(mm.write(b""), 0) + self.assertEqual(mm.write(b"x"), 1) + self.assertEqual(mm.write(b"yz"), 2) + self.assertEqual(mm.write(b"python"), 6) + + class LargeMmapTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS b/Misc/NEWS index 8770705..eea0b33 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -198,6 +198,9 @@ Core and Builtins Library ------- +- Issue #26335: Make mmap.write() return the number of bytes written like + other write methods. Patch by Jakub Stasiak. + - Issue #26457: Fixed the subnets() methods in IP network classes for the case when resulting prefix length is equal to maximal prefix length. Based on patch by Xiang Zhang. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index bb98a99..50cec24 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -389,6 +389,7 @@ mmap_write_method(mmap_object *self, PyObject *args) { Py_buffer data; + PyObject *result; CHECK_VALID(NULL); if (!PyArg_ParseTuple(args, "y*:write", &data)) @@ -406,9 +407,9 @@ mmap_write_method(mmap_object *self, } memcpy(self->data + self->pos, data.buf, data.len); self->pos = self->pos + data.len; + result = PyLong_FromSsize_t(data.len); PyBuffer_Release(&data); - Py_INCREF(Py_None); - return Py_None; + return result; } static PyObject * -- cgit v0.12 From d79d5b1a50347b06397e160440e6bb9dd5a8ed43 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 4 Mar 2016 09:55:07 -0800 Subject: More logicial order. Move space saving step to just before it is used. --- Modules/_collectionsmodule.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 7dcd7e7..19a86d1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -386,6 +386,13 @@ deque_extend(dequeobject *deque, PyObject *iterable) return result; } + it = PyObject_GetIter(iterable); + if (it == NULL) + return NULL; + + if (maxlen == 0) + return consume_iterator(it); + /* Space saving heuristic. Start filling from the left */ if (Py_SIZE(deque) == 0) { assert(deque->leftblock == deque->rightblock); @@ -394,13 +401,6 @@ deque_extend(dequeobject *deque, PyObject *iterable) deque->rightindex = 0; } - it = PyObject_GetIter(iterable); - if (it == NULL) - return NULL; - - if (maxlen == 0) - return consume_iterator(it); - iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque_append_internal(deque, item, maxlen) < 0) { @@ -433,6 +433,13 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) return result; } + it = PyObject_GetIter(iterable); + if (it == NULL) + return NULL; + + if (maxlen == 0) + return consume_iterator(it); + /* Space saving heuristic. Start filling from the right */ if (Py_SIZE(deque) == 0) { assert(deque->leftblock == deque->rightblock); @@ -441,13 +448,6 @@ deque_extendleft(dequeobject *deque, PyObject *iterable) deque->rightindex = BLOCKLEN - 2; } - it = PyObject_GetIter(iterable); - if (it == NULL) - return NULL; - - if (maxlen == 0) - return consume_iterator(it); - iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque_appendleft_internal(deque, item, maxlen) < 0) { -- cgit v0.12 From 841b930a41be90c6c05fb4bab8392f50e01d86b7 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 5 Mar 2016 14:05:45 +0200 Subject: Issue #17940: Remove redundant code from _Section.format_help() Output of func(*args) stored in the next line: item_help = join([func(*args) for func, args in self.items]) _Section.items only used by HelpFormatter._add_item() and it looks like it doesn't have any side effects. Patch by Yogesh Chaudhari. --- Lib/argparse.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index cc53841..209b4e9 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -210,8 +210,6 @@ class HelpFormatter(object): if self.parent is not None: self.formatter._indent() join = self.formatter._join_parts - for func, args in self.items: - func(*args) item_help = join([func(*args) for func, args in self.items]) if self.parent is not None: self.formatter._dedent() -- cgit v0.12 From a0d416f0d186b5ee6ea6a898418871157a55b4f6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Mar 2016 08:55:21 +0200 Subject: Issue #26482: Allowed pickling recursive dequeues. --- Lib/test/test_deque.py | 68 +++++++++++++++++++++++--------------------- Misc/NEWS | 2 ++ Modules/_collectionsmodule.c | 36 +++++++++++------------ 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index e2c6b6c..634d71f 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -622,20 +622,22 @@ class TestBasic(unittest.TestCase): self.assertEqual(list(d), list(e)) def test_pickle(self): - d = deque(range(200)) - for i in range(pickle.HIGHEST_PROTOCOL + 1): - s = pickle.dumps(d, i) - e = pickle.loads(s) - self.assertNotEqual(id(d), id(e)) - self.assertEqual(list(d), list(e)) - -## def test_pickle_recursive(self): -## d = deque('abc') -## d.append(d) -## for i in range(pickle.HIGHEST_PROTOCOL + 1): -## e = pickle.loads(pickle.dumps(d, i)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(id(e), id(e[-1])) + for d in deque(range(200)), deque(range(200), 100): + for i in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(d, i) + e = pickle.loads(s) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(list(e), list(d)) + self.assertEqual(e.maxlen, d.maxlen) + + def test_pickle_recursive(self): + for d in deque('abc'), deque('abc', 3): + d.append(d) + for i in range(pickle.HIGHEST_PROTOCOL + 1): + e = pickle.loads(pickle.dumps(d, i)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(id(e[-1]), id(e)) + self.assertEqual(e.maxlen, d.maxlen) def test_iterator_pickle(self): data = deque(range(200)) @@ -827,24 +829,26 @@ class TestSubclass(unittest.TestCase): self.assertEqual(type(d), type(e)) self.assertEqual(list(d), list(e)) -## def test_pickle(self): -## d = Deque('abc') -## d.append(d) -## -## e = pickle.loads(pickle.dumps(d)) -## self.assertNotEqual(id(d), id(e)) -## self.assertEqual(type(d), type(e)) -## dd = d.pop() -## ee = e.pop() -## self.assertEqual(id(e), id(ee)) -## self.assertEqual(d, e) -## -## d.x = d -## e = pickle.loads(pickle.dumps(d)) -## self.assertEqual(id(e), id(e.x)) -## -## d = DequeWithBadIter('abc') -## self.assertRaises(TypeError, pickle.dumps, d) + def test_pickle_recursive(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for d in Deque('abc'), Deque('abc', 3): + d.append(d) + + e = pickle.loads(pickle.dumps(d, proto)) + self.assertNotEqual(id(e), id(d)) + self.assertEqual(type(e), type(d)) + self.assertEqual(e.maxlen, d.maxlen) + dd = d.pop() + ee = e.pop() + self.assertEqual(id(ee), id(e)) + self.assertEqual(e, d) + + d.x = d + e = pickle.loads(pickle.dumps(d, proto)) + self.assertEqual(id(e.x), id(e)) + + for d in DequeWithBadIter('abc'), DequeWithBadIter('abc', 2): + self.assertRaises(TypeError, pickle.dumps, d, proto) def test_weakref(self): d = deque('gallahad') diff --git a/Misc/NEWS b/Misc/NEWS index 3eee944..4ce8bf6 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,8 @@ Core and Builtins Library ------- +- Issue #26482: Allowed pickling recursive dequeues. + - Issue #26335: Make mmap.write() return the number of bytes written like other write methods. Patch by Jakub Stasiak. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 19a86d1..d57f1ba 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1296,31 +1296,31 @@ deque_traverse(dequeobject *deque, visitproc visit, void *arg) static PyObject * deque_reduce(dequeobject *deque) { - PyObject *dict, *result, *aslist; + PyObject *dict, *it; _Py_IDENTIFIER(__dict__); dict = _PyObject_GetAttrId((PyObject *)deque, &PyId___dict__); - if (dict == NULL) + if (dict == NULL) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } PyErr_Clear(); - aslist = PySequence_List((PyObject *)deque); - if (aslist == NULL) { - Py_XDECREF(dict); + dict = Py_None; + Py_INCREF(dict); + } + + it = PyObject_GetIter((PyObject *)deque); + if (it == NULL) { + Py_DECREF(dict); return NULL; } - if (dict == NULL) { - if (deque->maxlen < 0) - result = Py_BuildValue("O(O)", Py_TYPE(deque), aslist); - else - result = Py_BuildValue("O(On)", Py_TYPE(deque), aslist, deque->maxlen); - } else { - if (deque->maxlen < 0) - result = Py_BuildValue("O(OO)O", Py_TYPE(deque), aslist, Py_None, dict); - else - result = Py_BuildValue("O(On)O", Py_TYPE(deque), aslist, deque->maxlen, dict); + + if (deque->maxlen < 0) { + return Py_BuildValue("O()NN", Py_TYPE(deque), dict, it); + } + else { + return Py_BuildValue("O(()n)NN", Py_TYPE(deque), deque->maxlen, dict, it); } - Py_XDECREF(dict); - Py_DECREF(aslist); - return result; } PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); -- cgit v0.12 From 818e18dd94611a2f0f4b0015661488ef0cd867dc Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Mar 2016 14:56:57 +0200 Subject: Issue #26167: Minimized overhead in copy.copy() and copy.deepcopy(). Optimized copying and deepcopying bytearrays, NotImplemented, slices, short lists, tuples, dicts, sets. --- Lib/copy.py | 115 +++++++++++++++++++++----------------------------- Lib/test/test_copy.py | 55 +++++++++++++++++++++--- Misc/NEWS | 4 ++ 3 files changed, 102 insertions(+), 72 deletions(-) diff --git a/Lib/copy.py b/Lib/copy.py index 972b94a..f86040a 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -51,7 +51,6 @@ __getstate__() and __setstate__(). See the documentation for module import types import weakref from copyreg import dispatch_table -import builtins class Error(Exception): pass @@ -102,37 +101,33 @@ def copy(x): else: raise Error("un(shallow)copyable object of type %s" % cls) - return _reconstruct(x, rv, 0) + if isinstance(rv, str): + return x + return _reconstruct(x, None, *rv) _copy_dispatch = d = {} def _copy_immutable(x): return x -for t in (type(None), int, float, bool, str, tuple, - bytes, frozenset, type, range, - types.BuiltinFunctionType, type(Ellipsis), +for t in (type(None), int, float, bool, complex, str, tuple, + bytes, frozenset, type, range, slice, + types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented), types.FunctionType, weakref.ref): d[t] = _copy_immutable t = getattr(types, "CodeType", None) if t is not None: d[t] = _copy_immutable -for name in ("complex", "unicode"): - t = getattr(builtins, name, None) - if t is not None: - d[t] = _copy_immutable - -def _copy_with_constructor(x): - return type(x)(x) -for t in (list, dict, set): - d[t] = _copy_with_constructor - -def _copy_with_copy_method(x): - return x.copy() + +d[list] = list.copy +d[dict] = dict.copy +d[set] = set.copy +d[bytearray] = bytearray.copy + if PyStringMap is not None: - d[PyStringMap] = _copy_with_copy_method + d[PyStringMap] = PyStringMap.copy -del d +del d, t def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. @@ -179,7 +174,10 @@ def deepcopy(x, memo=None, _nil=[]): else: raise Error( "un(deep)copyable object of type %s" % cls) - y = _reconstruct(x, rv, 1, memo) + if isinstance(rv, str): + y = x + else: + y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: @@ -193,13 +191,11 @@ def _deepcopy_atomic(x, memo): return x d[type(None)] = _deepcopy_atomic d[type(Ellipsis)] = _deepcopy_atomic +d[type(NotImplemented)] = _deepcopy_atomic d[int] = _deepcopy_atomic d[float] = _deepcopy_atomic d[bool] = _deepcopy_atomic -try: - d[complex] = _deepcopy_atomic -except NameError: - pass +d[complex] = _deepcopy_atomic d[bytes] = _deepcopy_atomic d[str] = _deepcopy_atomic try: @@ -211,15 +207,16 @@ d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[weakref.ref] = _deepcopy_atomic -def _deepcopy_list(x, memo): +def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] memo[id(x)] = y + append = y.append for a in x: - y.append(deepcopy(a, memo)) + append(deepcopy(a, memo)) return y d[list] = _deepcopy_list -def _deepcopy_tuple(x, memo): +def _deepcopy_tuple(x, memo, deepcopy=deepcopy): y = [deepcopy(a, memo) for a in x] # We're not going to put the tuple in the memo, but it's still important we # check for it, in case the tuple contains recursive mutable structures. @@ -236,7 +233,7 @@ def _deepcopy_tuple(x, memo): return y d[tuple] = _deepcopy_tuple -def _deepcopy_dict(x, memo): +def _deepcopy_dict(x, memo, deepcopy=deepcopy): y = {} memo[id(x)] = y for key, value in x.items(): @@ -248,7 +245,9 @@ if PyStringMap is not None: def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) -_deepcopy_dispatch[types.MethodType] = _deepcopy_method +d[types.MethodType] = _deepcopy_method + +del d def _keep_alive(x, memo): """Keeps a reference to the object x in the memo. @@ -266,31 +265,15 @@ def _keep_alive(x, memo): # aha, this is the first one :-) memo[id(memo)]=[x] -def _reconstruct(x, info, deep, memo=None): - if isinstance(info, str): - return x - assert isinstance(info, tuple) - if memo is None: - memo = {} - n = len(info) - assert n in (2, 3, 4, 5) - callable, args = info[:2] - if n > 2: - state = info[2] - else: - state = None - if n > 3: - listiter = info[3] - else: - listiter = None - if n > 4: - dictiter = info[4] - else: - dictiter = None +def _reconstruct(x, memo, func, args, + state=None, listiter=None, dictiter=None, + deepcopy=deepcopy): + deep = memo is not None + if deep and args: + args = (deepcopy(arg, memo) for arg in args) + y = func(*args) if deep: - args = deepcopy(args, memo) - y = callable(*args) - memo[id(x)] = y + memo[id(x)] = y if state is not None: if deep: @@ -309,22 +292,22 @@ def _reconstruct(x, info, deep, memo=None): setattr(y, key, value) if listiter is not None: - for item in listiter: - if deep: + if deep: + for item in listiter: item = deepcopy(item, memo) - y.append(item) + y.append(item) + else: + for item in listiter: + y.append(item) if dictiter is not None: - for key, value in dictiter: - if deep: + if deep: + for key, value in dictiter: key = deepcopy(key, memo) value = deepcopy(value, memo) - y[key] = value + y[key] = value + else: + for key, value in dictiter: + y[key] = value return y -del d - -del types - -# Helper for instance creation without calling __init__ -class _EmptyClass: - pass +del types, weakref, PyStringMap diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 7912c7c..45a6920 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -95,24 +95,67 @@ class TestCopy(unittest.TestCase): pass class WithMetaclass(metaclass=abc.ABCMeta): pass - tests = [None, 42, 2**100, 3.14, True, False, 1j, + tests = [None, ..., NotImplemented, + 42, 2**100, 3.14, True, False, 1j, "hello", "hello\u1234", f.__code__, - b"world", bytes(range(256)), - NewStyle, range(10), Classic, max, WithMetaclass] + b"world", bytes(range(256)), range(10), slice(1, 10, 2), + NewStyle, Classic, max, WithMetaclass] for x in tests: self.assertIs(copy.copy(x), x) def test_copy_list(self): x = [1, 2, 3] - self.assertEqual(copy.copy(x), x) + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = [] + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) def test_copy_tuple(self): x = (1, 2, 3) - self.assertEqual(copy.copy(x), x) + self.assertIs(copy.copy(x), x) + x = () + self.assertIs(copy.copy(x), x) + x = (1, 2, 3, []) + self.assertIs(copy.copy(x), x) def test_copy_dict(self): x = {"foo": 1, "bar": 2} - self.assertEqual(copy.copy(x), x) + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = {} + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + + def test_copy_set(self): + x = {1, 2, 3} + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = set() + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + + def test_copy_frozenset(self): + x = frozenset({1, 2, 3}) + self.assertIs(copy.copy(x), x) + x = frozenset() + self.assertIs(copy.copy(x), x) + + def test_copy_bytearray(self): + x = bytearray(b'abc') + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) + x = bytearray() + y = copy.copy(x) + self.assertEqual(y, x) + self.assertIsNot(y, x) def test_copy_inst_vanilla(self): class C: diff --git a/Misc/NEWS b/Misc/NEWS index 3e10fe4..026db17 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,10 @@ Core and Builtins Library ------- +- Issue #26167: Minimized overhead in copy.copy() and copy.deepcopy(). + Optimized copying and deepcopying bytearrays, NotImplemented, slices, + short lists, tuples, dicts, sets. + - Issue #25718: Fixed pickling and copying the accumulate() iterator with total is None. -- cgit v0.12 From a2998a63c8951ab8254b96860fe64dcaa215eb1f Mon Sep 17 00:00:00 2001 From: Alexander Belopolsky Date: Sun, 6 Mar 2016 14:58:43 -0500 Subject: Closes #19475: Added timespec to the datetime.isoformat() method. Added an optional argument timespec to the datetime isoformat() method to choose the precision of the time component. Original patch by Alessandro Cucci. --- Doc/library/datetime.rst | 70 ++++++++++++++++++++++++-- Lib/datetime.py | 51 +++++++++++++------ Lib/test/datetimetester.py | 48 +++++++++++++++--- Misc/ACKS | 1 + Misc/NEWS | 3 ++ Modules/_datetimemodule.c | 123 ++++++++++++++++++++++++++++++++++----------- 6 files changed, 244 insertions(+), 52 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index a822842..d73e97b 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1134,7 +1134,7 @@ Instance methods: ``self.date().isocalendar()``. -.. method:: datetime.isoformat(sep='T') +.. method:: datetime.isoformat(sep='T', timespec='auto') Return a string representing the date and time in ISO 8601 format, YYYY-MM-DDTHH:MM:SS.mmmmmm or, if :attr:`microsecond` is 0, @@ -1155,6 +1155,37 @@ Instance methods: >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' + The optional argument *timespec* specifies the number of additional + components of the time to include (the default is ``'auto'``). + It can be one of the following: + + - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, + same as ``'microseconds'`` otherwise. + - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` + in HH:MM:SS format. + - ``'milliseconds'``: Include full time, but truncate fractional second + part to milliseconds. HH:MM:SS.sss format. + - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + + .. note:: + + Excluded time components are truncated, not rounded. + + :exc:`ValueError` will be raised on an invalid *timespec* argument. + + + >>> from datetime import datetime + >>> datetime.now().isoformat(timespec='minutes') + '2002-12-25T00:00' + >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) + >>> dt.isoformat(timespec='microseconds') + '2015-01-01T12:30:59.000000' + + .. versionadded:: 3.6 + Added the *timespec* argument. + .. method:: datetime.__str__() @@ -1404,13 +1435,46 @@ Instance methods: aware :class:`.time`, without conversion of the time data. -.. method:: time.isoformat() +.. method:: time.isoformat(timespec='auto') Return a string representing the time in ISO 8601 format, HH:MM:SS.mmmmmm or, if - self.microsecond is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a + :attr:`microsecond` is 0, HH:MM:SS If :meth:`utcoffset` does not return ``None``, a 6-character string is appended, giving the UTC offset in (signed) hours and minutes: HH:MM:SS.mmmmmm+HH:MM or, if self.microsecond is 0, HH:MM:SS+HH:MM + The optional argument *timespec* specifies the number of additional + components of the time to include (the default is ``'auto'``). + It can be one of the following: + + - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, + same as ``'microseconds'`` otherwise. + - ``'hours'``: Include the :attr:`hour` in the two-digit HH format. + - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in HH:MM format. + - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` + in HH:MM:SS format. + - ``'milliseconds'``: Include full time, but truncate fractional second + part to milliseconds. HH:MM:SS.sss format. + - ``'microseconds'``: Include full time in HH:MM:SS.mmmmmm format. + + .. note:: + + Excluded time components are truncated, not rounded. + + :exc:`ValueError` will be raised on an invalid *timespec* argument. + + + >>> from datetime import time + >>> time(hours=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + '12:34' + >>> dt = time(hours=12, minute=34, second=56, microsecond=0) + >>> dt.isoformat(timespec='microseconds') + '12:34:56.000000' + >>> dt.isoformat(timespec='auto') + '12:34:56' + + .. versionadded:: 3.6 + Added the *timespec* argument. + .. method:: time.__str__() diff --git a/Lib/datetime.py b/Lib/datetime.py index 3206923..d4b7fc4 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -152,12 +152,26 @@ def _build_struct_time(y, m, d, hh, mm, ss, dstflag): dnum = _days_before_month(y, m) + d return _time.struct_time((y, m, d, hh, mm, ss, wday, dnum, dstflag)) -def _format_time(hh, mm, ss, us): - # Skip trailing microseconds when us==0. - result = "%02d:%02d:%02d" % (hh, mm, ss) - if us: - result += ".%06d" % us - return result +def _format_time(hh, mm, ss, us, timespec='auto'): + specs = { + 'hours': '{:02d}', + 'minutes': '{:02d}:{:02d}', + 'seconds': '{:02d}:{:02d}:{:02d}', + 'milliseconds': '{:02d}:{:02d}:{:02d}.{:03d}', + 'microseconds': '{:02d}:{:02d}:{:02d}.{:06d}' + } + + if timespec == 'auto': + # Skip trailing microseconds when us==0. + timespec = 'microseconds' if us else 'seconds' + elif timespec == 'milliseconds': + us //= 1000 + try: + fmt = specs[timespec] + except KeyError: + raise ValueError('Unknown timespec value') + else: + return fmt.format(hh, mm, ss, us) # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): @@ -1194,14 +1208,17 @@ class time: s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")" return s - def isoformat(self): + def isoformat(self, timespec='auto'): """Return the time formatted according to ISO. - This is 'HH:MM:SS.mmmmmm+zz:zz', or 'HH:MM:SS+zz:zz' if - self.microsecond == 0. + The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional + part is omitted if self.microsecond == 0. + + The optional argument timespec specifies the number of additional + terms of the time to include. """ s = _format_time(self._hour, self._minute, self._second, - self._microsecond) + self._microsecond, timespec) tz = self._tzstr() if tz: s += tz @@ -1550,21 +1567,25 @@ class datetime(date): self._hour, self._minute, self._second, self._year) - def isoformat(self, sep='T'): + def isoformat(self, sep='T', timespec='auto'): """Return the time formatted according to ISO. - This is 'YYYY-MM-DD HH:MM:SS.mmmmmm', or 'YYYY-MM-DD HH:MM:SS' if - self.microsecond == 0. + The full format looks like 'YYYY-MM-DD HH:MM:SS.mmmmmm'. + By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM' or 'YYYY-MM-DD HH:MM:SS+HH:MM'. + giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. + + The optional argument timespec specifies the number of additional + terms of the time to include. """ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + _format_time(self._hour, self._minute, self._second, - self._microsecond)) + self._microsecond, timespec)) + off = self.utcoffset() if off is not None: if off.days < 0: diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 68c18bd..b49942a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1556,13 +1556,32 @@ class TestDateTime(TestDate): self.assertEqual(dt, dt2) def test_isoformat(self): - t = self.theclass(2, 3, 2, 4, 5, 1, 123) - self.assertEqual(t.isoformat(), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat('T'), "0002-03-02T04:05:01.000123") - self.assertEqual(t.isoformat(' '), "0002-03-02 04:05:01.000123") - self.assertEqual(t.isoformat('\x00'), "0002-03-02\x0004:05:01.000123") + t = self.theclass(1, 2, 3, 4, 5, 1, 123) + self.assertEqual(t.isoformat(), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat('T'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(' '), "0001-02-03 04:05:01.000123") + self.assertEqual(t.isoformat('\x00'), "0001-02-03\x0004:05:01.000123") + self.assertEqual(t.isoformat(timespec='hours'), "0001-02-03T04") + self.assertEqual(t.isoformat(timespec='minutes'), "0001-02-03T04:05") + self.assertEqual(t.isoformat(timespec='seconds'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01.000123") + self.assertEqual(t.isoformat(sep=' ', timespec='minutes'), "0001-02-03 04:05") + self.assertRaises(ValueError, t.isoformat, timespec='foo') # str is ISO format with the separator forced to a blank. - self.assertEqual(str(t), "0002-03-02 04:05:01.000123") + self.assertEqual(str(t), "0001-02-03 04:05:01.000123") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500, tzinfo=timezone.utc) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999+00:00") + + t = self.theclass(1, 2, 3, 4, 5, 1, 999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.999") + + t = self.theclass(1, 2, 3, 4, 5, 1) + self.assertEqual(t.isoformat(timespec='auto'), "0001-02-03T04:05:01") + self.assertEqual(t.isoformat(timespec='milliseconds'), "0001-02-03T04:05:01.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "0001-02-03T04:05:01.000000") t = self.theclass(2, 3, 2) self.assertEqual(t.isoformat(), "0002-03-02T00:00:00") @@ -2322,6 +2341,23 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(t.isoformat(), "00:00:00.100000") self.assertEqual(t.isoformat(), str(t)) + t = self.theclass(hour=12, minute=34, second=56, microsecond=123456) + self.assertEqual(t.isoformat(timespec='hours'), "12") + self.assertEqual(t.isoformat(timespec='minutes'), "12:34") + self.assertEqual(t.isoformat(timespec='seconds'), "12:34:56") + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.123") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.123456") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56.123456") + self.assertRaises(ValueError, t.isoformat, timespec='monkey') + + t = self.theclass(hour=12, minute=34, second=56, microsecond=999500) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.999") + + t = self.theclass(hour=12, minute=34, second=56, microsecond=0) + self.assertEqual(t.isoformat(timespec='milliseconds'), "12:34:56.000") + self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.000000") + self.assertEqual(t.isoformat(timespec='auto'), "12:34:56") + def test_1653736(self): # verify it doesn't accept extra keyword arguments t = self.theclass(second=1) diff --git a/Misc/ACKS b/Misc/ACKS index 5e6fee7..adc6848 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -309,6 +309,7 @@ Laura Creighton Simon Cross Felipe Cruz Drew Csillag +Alessandro Cucci Joaquin Cuenca Abela John Cugini Tom Culliton diff --git a/Misc/NEWS b/Misc/NEWS index 9c94db3..d90422d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #19475: Added an optional argument timespec to the datetime + isoformat() method to choose the precision of the time component. + - Issue #2202: Fix UnboundLocalError in AbstractDigestAuthHandler.get_algorithm_impls. Initial patch by Mathieu Dupuy. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 4c8f3f6..347a9d9 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3608,23 +3608,56 @@ time_str(PyDateTime_Time *self) } static PyObject * -time_isoformat(PyDateTime_Time *self, PyObject *unused) +time_isoformat(PyDateTime_Time *self, PyObject *args, PyObject *kw) { char buf[100]; + char *timespec = NULL; + static char *keywords[] = {"timespec", NULL}; PyObject *result; int us = TIME_GET_MICROSECOND(self); + static char *specs[][2] = { + {"hours", "%02d"}, + {"minutes", "%02d:%02d"}, + {"seconds", "%02d:%02d:%02d"}, + {"milliseconds", "%02d:%02d:%02d.%03d"}, + {"microseconds", "%02d:%02d:%02d.%06d"}, + }; + size_t given_spec; - if (us) - result = PyUnicode_FromFormat("%02d:%02d:%02d.%06d", - TIME_GET_HOUR(self), - TIME_GET_MINUTE(self), - TIME_GET_SECOND(self), - us); - else - result = PyUnicode_FromFormat("%02d:%02d:%02d", - TIME_GET_HOUR(self), - TIME_GET_MINUTE(self), - TIME_GET_SECOND(self)); + if (!PyArg_ParseTupleAndKeywords(args, kw, "|s:isoformat", keywords, ×pec)) + return NULL; + + if (timespec == NULL || strcmp(timespec, "auto") == 0) { + if (us == 0) { + /* seconds */ + given_spec = 2; + } + else { + /* microseconds */ + given_spec = 4; + } + } + else { + for (given_spec = 0; given_spec < Py_ARRAY_LENGTH(specs); given_spec++) { + if (strcmp(timespec, specs[given_spec][0]) == 0) { + if (given_spec == 3) { + /* milliseconds */ + us = us / 1000; + } + break; + } + } + } + + if (given_spec == Py_ARRAY_LENGTH(specs)) { + PyErr_Format(PyExc_ValueError, "Unknown timespec value"); + return NULL; + } + else { + result = PyUnicode_FromFormat(specs[given_spec][1], + TIME_GET_HOUR(self), TIME_GET_MINUTE(self), + TIME_GET_SECOND(self), us); + } if (result == NULL || !HASTZINFO(self) || self->tzinfo == Py_None) return result; @@ -3845,9 +3878,10 @@ time_reduce(PyDateTime_Time *self, PyObject *arg) static PyMethodDef time_methods[] = { - {"isoformat", (PyCFunction)time_isoformat, METH_NOARGS, - PyDoc_STR("Return string in ISO 8601 format, HH:MM:SS[.mmmmmm]" - "[+HH:MM].")}, + {"isoformat", (PyCFunction)time_isoformat, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Return string in ISO 8601 format, [HH[:MM[:SS[.mmm[uuu]]]]]" + "[+HH:MM].\n\n" + "timespec specifies what components of the time to include.\n")}, {"strftime", (PyCFunction)time_strftime, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("format -> strftime() style string.")}, @@ -4476,25 +4510,55 @@ static PyObject * datetime_isoformat(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) { int sep = 'T'; - static char *keywords[] = {"sep", NULL}; + char *timespec = NULL; + static char *keywords[] = {"sep", "timespec", NULL}; char buffer[100]; - PyObject *result; + PyObject *result = NULL; int us = DATE_GET_MICROSECOND(self); + static char *specs[][2] = { + {"hours", "%04d-%02d-%02d%c%02d"}, + {"minutes", "%04d-%02d-%02d%c%02d:%02d"}, + {"seconds", "%04d-%02d-%02d%c%02d:%02d:%02d"}, + {"milliseconds", "%04d-%02d-%02d%c%02d:%02d:%02d.%03d"}, + {"microseconds", "%04d-%02d-%02d%c%02d:%02d:%02d.%06d"}, + }; + size_t given_spec; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|C:isoformat", keywords, &sep)) + if (!PyArg_ParseTupleAndKeywords(args, kw, "|Cs:isoformat", keywords, &sep, ×pec)) return NULL; - if (us) - result = PyUnicode_FromFormat("%04d-%02d-%02d%c%02d:%02d:%02d.%06d", + + if (timespec == NULL || strcmp(timespec, "auto") == 0) { + if (us == 0) { + /* seconds */ + given_spec = 2; + } + else { + /* microseconds */ + given_spec = 4; + } + } + else { + for (given_spec = 0; given_spec < Py_ARRAY_LENGTH(specs); given_spec++) { + if (strcmp(timespec, specs[given_spec][0]) == 0) { + if (given_spec == 3) { + us = us / 1000; + } + break; + } + } + } + + if (given_spec == Py_ARRAY_LENGTH(specs)) { + PyErr_Format(PyExc_ValueError, "Unknown timespec value"); + return NULL; + } + else { + result = PyUnicode_FromFormat(specs[given_spec][1], GET_YEAR(self), GET_MONTH(self), GET_DAY(self), (int)sep, DATE_GET_HOUR(self), DATE_GET_MINUTE(self), DATE_GET_SECOND(self), us); - else - result = PyUnicode_FromFormat("%04d-%02d-%02d%c%02d:%02d:%02d", - GET_YEAR(self), GET_MONTH(self), - GET_DAY(self), (int)sep, - DATE_GET_HOUR(self), DATE_GET_MINUTE(self), - DATE_GET_SECOND(self)); + } if (!result || !HASTZINFO(self)) return result; @@ -5028,9 +5092,12 @@ static PyMethodDef datetime_methods[] = { {"isoformat", (PyCFunction)datetime_isoformat, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("[sep] -> string in ISO 8601 format, " - "YYYY-MM-DDTHH:MM:SS[.mmmmmm][+HH:MM].\n\n" + "YYYY-MM-DDT[HH[:MM[:SS[.mmm[uuu]]]]][+HH:MM].\n" "sep is used to separate the year from the time, and " - "defaults to 'T'.")}, + "defaults to 'T'.\n" + "timespec specifies what components of the time to include" + " (allowed values are 'auto', 'hours', 'minutes', 'seconds'," + " 'milliseconds', and 'microseconds').\n")}, {"utcoffset", (PyCFunction)datetime_utcoffset, METH_NOARGS, PyDoc_STR("Return self.tzinfo.utcoffset(self).")}, -- cgit v0.12 From b9f3114d42601b07c9830af8b18330bff475bcfc Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 10 Mar 2016 01:06:23 +0000 Subject: Issue #21042: Return full path in ctypes.util.find_library() on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patch by Tamás Bence Gedai. --- Doc/library/ctypes.rst | 11 +++++++---- Lib/ctypes/test/test_find.py | 40 ++++++++++++++++++++++++---------------- Lib/ctypes/util.py | 4 ++-- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3b0c956..58e1ea3 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1252,15 +1252,15 @@ The exact functionality is system dependent. On Linux, :func:`find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, and ``objdump``) to find the library file. It -returns the filename of the library file. Here are some examples:: +returns the absolute path of the library file. Here are some examples:: >>> from ctypes.util import find_library >>> find_library("m") - 'libm.so.6' + '/lib/x86_64-linux-gnu/libm.so.6' >>> find_library("c") - 'libc.so.6' + '/lib/x86_64-linux-gnu/libc.so.6' >>> find_library("bz2") - 'libbz2.so.1.0' + '/lib/x86_64-linux-gnu/libbz2.so.1.0' >>> On OS X, :func:`find_library` tries several predefined naming schemes and paths @@ -1829,6 +1829,9 @@ Utility functions The exact functionality is system dependent. + .. versionchanged:: 3.6 + On Linux it returns an absolute path. + .. function:: find_msvcrt() :module: ctypes.util diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index e6bc19d..1845bb0 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -9,39 +9,39 @@ from ctypes.util import find_library class Test_OpenGL_libs(unittest.TestCase): @classmethod def setUpClass(cls): - lib_gl = lib_glu = lib_gle = None + cls.lib_gl = cls.lib_glu = cls.lib_gle = None if sys.platform == "win32": - lib_gl = find_library("OpenGL32") - lib_glu = find_library("Glu32") + cls.lib_gl = find_library("OpenGL32") + cls.lib_glu = find_library("Glu32") elif sys.platform == "darwin": - lib_gl = lib_glu = find_library("OpenGL") + cls.lib_gl = cls.lib_glu = find_library("OpenGL") else: - lib_gl = find_library("GL") - lib_glu = find_library("GLU") - lib_gle = find_library("gle") + cls.lib_gl = find_library("GL") + cls.lib_glu = find_library("GLU") + cls.lib_gle = find_library("gle") ## print, for debugging if test.support.verbose: print("OpenGL libraries:") - for item in (("GL", lib_gl), - ("GLU", lib_glu), - ("gle", lib_gle)): + for item in (("GL", cls.lib_gl), + ("GLU", cls.lib_glu), + ("gle", cls.lib_gle)): print("\t", item) cls.gl = cls.glu = cls.gle = None - if lib_gl: + if cls.lib_gl: try: - cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) + cls.gl = CDLL(cls.lib_gl, mode=RTLD_GLOBAL) except OSError: pass - if lib_glu: + if cls.lib_glu: try: - cls.glu = CDLL(lib_glu, RTLD_GLOBAL) + cls.glu = CDLL(cls.lib_glu, RTLD_GLOBAL) except OSError: pass - if lib_gle: + if cls.lib_gle: try: - cls.gle = CDLL(lib_gle) + cls.gle = CDLL(cls.lib_gle) except OSError: pass @@ -64,6 +64,14 @@ class Test_OpenGL_libs(unittest.TestCase): self.skipTest('lib_gle not available') self.gle.gleGetJoinStyle + def test_abspath(self): + if self.lib_gl: + self.assertTrue(os.path.isabs(self.lib_gl)) + if self.lib_glu: + self.assertTrue(os.path.isabs(self.lib_glu)) + if self.lib_gle: + self.assertTrue(os.path.isabs(self.lib_gle)) + # On platforms where the default shared library suffix is '.so', # at least some libraries can be loaded as attributes of the cdll # object, since ctypes now tries loading the lib again diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 9e74ccd..d8e3bfa 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -221,8 +221,8 @@ elif os.name == "posix": abi_type = mach_map.get(machine, 'libc6') # XXX assuming GLIBC's ldconfig (with option -p) - regex = os.fsencode( - '\s+(lib%s\.[^\s]+)\s+\(%s' % (re.escape(name), abi_type)) + regex = r'lib%s\.[^\s]+\s\(%s(?:,\s.*)?\)\s=>\s(.*)' + regex = os.fsencode(regex % (re.escape(name), abi_type)) try: with subprocess.Popen(['/sbin/ldconfig', '-p'], stdin=subprocess.DEVNULL, diff --git a/Misc/ACKS b/Misc/ACKS index 6d8fcf8..0a67f39 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -487,6 +487,7 @@ Matthieu Gautier Stephen M. Gava Xavier de Gaye Harry Henry Gebel +Tamás Bence Gedai Marius Gedminas Jan-Philip Gehrcke Thomas Gellekum diff --git a/Misc/NEWS b/Misc/NEWS index 205891b..d678a69 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #21042: Make ctypes.util.find_library() return the full path on + Linux, similar to other platforms. Patch by Tamás Bence Gedai. + - Issue #15068: Got rid of excessive buffering in fileinput. The bufsize parameter is now deprecated and ignored. -- cgit v0.12 From 2c2a4e63d794eb55e9163322ea11b9765e9e0db5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 11 Mar 2016 22:17:48 +0100 Subject: Add Mock.assert_called() Issue #26323: Add assert_called() and assert_called_once() methods to unittest.mock.Mock. --- Doc/library/unittest.mock.rst | 28 ++++++++++++++++++++++++++++ Doc/whatsnew/3.6.rst | 12 ++++++++++++ Lib/unittest/mock.py | 18 ++++++++++++++++++ Lib/unittest/test/testmock/testmock.py | 21 +++++++++++++++++++++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ 6 files changed, 83 insertions(+) diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 9a51194..c4dc4ed 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -259,6 +259,34 @@ the *new_callable* argument to :func:`patch`. used to set attributes on the mock after it is created. See the :meth:`configure_mock` method for details. + .. method:: assert_called(*args, **kwargs) + + Assert that the mock was called at least once. + + >>> mock = Mock() + >>> mock.method() + + >>> mock.method.assert_called() + + .. versionadded:: 3.6 + + .. method:: assert_called_once(*args, **kwargs) + + Assert that the mock was called exactly once. + + >>> mock = Mock() + >>> mock.method() + + >>> mock.method.assert_called_once() + >>> mock.method() + + >>> mock.method.assert_called_once() + Traceback (most recent call last): + ... + AssertionError: Expected 'method' to have been called once. Called 2 times. + + .. versionadded:: 3.6 + .. method:: assert_called_with(*args, **kwargs) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 5c02b7d..3afe2d4 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -161,6 +161,18 @@ telnetlib Stéphane Wirtel in :issue:`25485`). +unittest.mock +------------- + +The :class:`~unittest.mock.Mock` class has the following improvements: + +* Two new methods, :meth:`Mock.assert_called() + ` and :meth:`Mock.assert_called_once() + ` to check if the mock object + was called. + (Contributed by Amit Saha in :issue:`26323`.) + + urllib.robotparser ------------------ diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 21f49fa..50ff949 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -772,6 +772,24 @@ class NonCallableMock(Base): (self._mock_name or 'mock', self.call_count)) raise AssertionError(msg) + def assert_called(_mock_self): + """assert that the mock was called at least once + """ + self = _mock_self + if self.call_count == 0: + msg = ("Expected '%s' to have been called." % + self._mock_name or 'mock') + raise AssertionError(msg) + + def assert_called_once(_mock_self): + """assert that the mock was called only once. + """ + self = _mock_self + if not self.call_count == 1: + msg = ("Expected '%s' to have been called once. Called %s times." % + (self._mock_name or 'mock', self.call_count)) + raise AssertionError(msg) + def assert_called_with(_mock_self, *args, **kwargs): """assert that the mock was called with the specified arguments. diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py index 2a6069f..b2d9acb 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -1222,6 +1222,27 @@ class MockTest(unittest.TestCase): with self.assertRaises(AssertionError): m.hello.assert_not_called() + def test_assert_called(self): + m = Mock() + with self.assertRaises(AssertionError): + m.hello.assert_called() + m.hello() + m.hello.assert_called() + + m.hello() + m.hello.assert_called() + + def test_assert_called_once(self): + m = Mock() + with self.assertRaises(AssertionError): + m.hello.assert_called_once() + m.hello() + m.hello.assert_called_once() + + m.hello() + with self.assertRaises(AssertionError): + m.hello.assert_called_once() + #Issue21256 printout of keyword args should be in deterministic order def test_sorted_call_signature(self): m = Mock() diff --git a/Misc/ACKS b/Misc/ACKS index 0a67f39..a19e113 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1267,6 +1267,7 @@ Bernt Røskar Brenna Constantina S. Patrick Sabin Sébastien Sablé +Amit Saha Suman Saha Hajime Saitou George Sakkis diff --git a/Misc/NEWS b/Misc/NEWS index 45d8f2e..1e87de8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -201,6 +201,9 @@ Core and Builtins Library ------- +- Issue #26323: Add Mock.assert_called() and Mock.assert_called_once() + methods to unittest.mock. Patch written by Amit Saha. + - Issue #20589: Invoking Path.owner() and Path.group() on Windows now raise NotImplementedError instead of ImportError. -- cgit v0.12 From 474ebbbe50ea3247ad120324874b8b32d5fa550d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 11 Mar 2016 22:36:14 +0100 Subject: Always test datetime.strftime("%4Y") Issue #13305: Always test datetime.datetime.strftime("%4Y") for years < 1900. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b49942a..0be1af4 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1422,8 +1422,8 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): if d.strftime("%Y") != '%04d' % y: # Year 42 returns '42', not padded self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # '0042' is obtained anyway + self.assertEqual(d.strftime("%4Y"), '%04d' % y) def test_replace(self): cls = self.theclass -- cgit v0.12 From 15a83e845fa2421428f9d06c6fe6e85fde59d22c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2016 08:16:48 +0100 Subject: Revert change 291d47954618 Revert: "Always test datetime.strftime("%4Y") Issue #13305: Always test datetime.datetime.strftime("%4Y") for years < 1900." In fact, strftime("%4Y") fails on most platforms. --- Lib/test/datetimetester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 0be1af4..b49942a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1422,8 +1422,8 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): if d.strftime("%Y") != '%04d' % y: # Year 42 returns '42', not padded self.assertEqual(d.strftime("%Y"), '%d' % y) - # '0042' is obtained anyway - self.assertEqual(d.strftime("%4Y"), '%04d' % y) + # '0042' is obtained anyway + self.assertEqual(d.strftime("%4Y"), '%04d' % y) def test_replace(self): cls = self.theclass -- cgit v0.12 From 04bc5b9e4864a91f4dbea74ce2d88f606aa578f3 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 14 Mar 2016 06:06:03 +0200 Subject: Issue #747320: Use email.utils.formatdate() to avoid code duplication in BaseHTTPRequestHandler Initial patch by karlcow. --- Lib/http/server.py | 8 ++------ Lib/test/test_httpservers.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index e1b71ab..5e91826 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -87,6 +87,7 @@ __all__ = [ "SimpleHTTPRequestHandler", "CGIHTTPRequestHandler", ] +import email.utils import html import http.client import io @@ -566,12 +567,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): """Return the current date and time formatted for a message header.""" if timestamp is None: timestamp = time.time() - year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp) - s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( - self.weekdayname[wd], - day, self.monthname[month], year, - hh, mm, ss) - return s + return email.utils.formatdate(timestamp, usegmt=True) def log_date_time_string(self): """Return the current time formatted for logging.""" diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index d4c8eab..5d44825 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -17,6 +17,7 @@ import urllib.parse import html import http.client import tempfile +import time from io import BytesIO import unittest @@ -873,6 +874,19 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.handler.handle() self.assertRaises(StopIteration, next, close_values) + def test_date_time_string(self): + now = time.time() + # this is the old code that formats the timestamp + year, month, day, hh, mm, ss, wd, y, z = time.gmtime(now) + expected = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % ( + self.handler.weekdayname[wd], + day, + self.handler.monthname[month], + year, hh, mm, ss + ) + self.assertEqual(self.handler.date_time_string(timestamp=now), expected) + + class SimpleHTTPRequestHandlerTestCase(unittest.TestCase): """ Test url parsing """ def setUp(self): -- cgit v0.12 From 34be807ca4dfecc5b0a9e577a48535e738dce32b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 12:04:26 +0100 Subject: Add PYTHONMALLOC env var Issue #26516: * Add PYTHONMALLOC environment variable to set the Python memory allocators and/or install debug hooks. * PyMem_SetupDebugHooks() can now also be used on Python compiled in release mode. * The PYTHONMALLOCSTATS environment variable can now also be used on Python compiled in release mode. It now has no effect if set to an empty string. * In debug mode, debug hooks are now also installed on Python memory allocators when Python is configured without pymalloc. --- Doc/c-api/memory.rst | 38 ++++++--- Doc/using/cmdline.rst | 51 +++++++++++-- Doc/whatsnew/3.6.rst | 31 ++++++++ Include/pymem.h | 9 +++ Lib/test/test_capi.py | 59 ++++++++++++++ Misc/NEWS | 13 ++++ Misc/README.valgrind | 3 + Modules/_testcapimodule.c | 29 +++++++ Modules/main.c | 25 ++++-- Objects/obmalloc.c | 191 ++++++++++++++++++++++++++++++++-------------- Programs/python.c | 9 +++ Python/pylifecycle.c | 9 ++- Python/sysmodule.c | 6 +- 13 files changed, 383 insertions(+), 90 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 290ef09..fe1cd5f 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -85,9 +85,12 @@ for the I/O buffer escapes completely the Python memory manager. .. seealso:: + The :envvar:`PYTHONMALLOC` environment variable can be used to configure + the memory allocators used by Python. + The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - memory allocation statistics every time a new object arena is created, and - on shutdown. + statistics of the :ref:`pymalloc memory allocator ` every time a + new pymalloc object arena is created, and on shutdown. Raw Memory Interface @@ -343,25 +346,36 @@ Customize Memory Allocators - detect write before the start of the buffer (buffer underflow) - detect write after the end of the buffer (buffer overflow) - The function does nothing if Python is not compiled is debug mode. + These hooks are installed by default if Python is compiled in debug + mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install + debug hooks on a Python compiled in release mode. + + .. versionchanged:: 3.6 + This function now also works on Python compiled in release mode. + +.. _pymalloc: -Customize PyObject Arena Allocator -================================== +The pymalloc allocator +====================== -Python has a *pymalloc* allocator for allocations smaller than 512 bytes. This -allocator is optimized for small objects with a short lifetime. It uses memory -mappings called "arenas" with a fixed size of 256 KB. It falls back to -:c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger -than 512 bytes. *pymalloc* is the default allocator used by -:c:func:`PyObject_Malloc`. +Python has a *pymalloc* allocator optimized for small objects (smaller or equal +to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" +with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and +:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. -The default arena allocator uses the following functions: +*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_OBJ` domain +(:c:func:`PyObject_Malloc` & cie). + +The arena allocator uses the following functions: * :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, * :c:func:`mmap` and :c:func:`munmap` if available, * :c:func:`malloc` and :c:func:`free` otherwise. +Customize pymalloc Arena Allocator +---------------------------------- + .. versionadded:: 3.4 .. c:type:: PyObjectArenaAllocator diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index ec744a3..684ccb6 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -621,6 +621,51 @@ conflict. .. versionadded:: 3.4 +.. envvar:: PYTHONMALLOC + + Set the Python memory allocators and/or install debug hooks. + + Set the family of memory allocators used by Python: + + * ``malloc``: use the :c:func:`malloc` function of the C library + for all Python memory allocators (:c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc`, :c:func:`PyObject_Malloc` & cie). + * ``pymalloc``: :c:func:`PyObject_Malloc`, :c:func:`PyObject_Calloc` and + :c:func:`PyObject_Realloc` use the :ref:`pymalloc allocator `. + Other Python memory allocators (:c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc` & cie) use :c:func:`malloc`. + + Install debug hooks: + + * ``debug``: install debug hooks on top of the default memory allocator + * ``malloc_debug``: same than ``malloc`` but also install debug hooks + * ``pymalloc_debug``: same than ``malloc`` but also install debug hooks + + See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python + memory allocators. + + .. note:: + ``pymalloc`` and ``pymalloc_debug`` are not available if Python is + configured without ``pymalloc`` support. + + .. versionadded:: 3.6 + + +.. envvar:: PYTHONMALLOCSTATS + + If set to a non-empty string, Python will print statistics of the + :ref:`pymalloc memory allocator ` every time a new pymalloc object + arena is created, and on shutdown. + + This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable + is used to force the :c:func:`malloc` allocator of the C library, or if + Python is configured without ``pymalloc`` support. + + .. versionchanged:: 3.6 + This variable can now also be used on Python compiled in release mode. + It now has no effect if set to an empty string. + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ @@ -636,9 +681,3 @@ if Python was configured with the ``--with-pydebug`` build option. If set, Python will dump objects and reference counts still alive after shutting down the interpreter. - - -.. envvar:: PYTHONMALLOCSTATS - - If set, Python will print memory allocation statistics every time a new - object arena is created, and on shutdown. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3afe2d4..588826b 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -80,6 +80,9 @@ Summary -- Release highlights PEP written by Carl Meyer +New Features +============ + .. _whatsnew-fstrings: PEP 498: Formatted string literals @@ -98,6 +101,34 @@ evaluated at run time, and then formatted using the :func:`format` protocol. See :pep:`498` and the main documentation at :ref:`f-strings`. +PYTHONMALLOC environment variable +--------------------------------- + +The new :envvar:`PYTHONMALLOC` environment variable allows to set the Python +memory allocators and/or install debug hooks. + +It is now possible to install debug hooks on Python memory allocators on Python +compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: + +* Newly allocated memory is filled with the byte ``0xCB`` +* Freed memory is filled with the byte ``0xDB`` +* Detect violations of Python memory allocator API. For example, + :c:func:`PyObject_Free` called on a memory block allocated by + :c:func:`PyMem_Malloc`. +* Detect write before the start of the buffer (buffer underflow) +* Detect write after the end of the buffer (buffer overflow) + +See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python +memory allocators. + +It is now also possible to force the usage of the :c:func:`malloc` allocator of +the C library for all Python memory allocations using ``PYTHONMALLOC=malloc``. +It helps to use external memory debuggers like Valgrind on a Python compiled in +release mode. + +(Contributed by Victor Stinner in :issue:`26516`.) + + Other Language Changes ====================== diff --git a/Include/pymem.h b/Include/pymem.h index 043db64..b1f06ef 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -16,8 +16,17 @@ PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_RawFree(void *ptr); + +/* Configure the Python memory allocators. Pass NULL to use default + allocators. */ +PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); + +#ifdef WITH_PYMALLOC +PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif +#endif /* !Py_LIMITED_API */ + /* BEWARE: diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 74ec6c5..d56d702 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -6,6 +6,7 @@ import pickle import random import subprocess import sys +import sysconfig import textwrap import time import unittest @@ -521,6 +522,7 @@ class SkipitemTest(unittest.TestCase): self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, (), {}, b'', [42]) + @unittest.skipUnless(threading, 'Threading required for this test.') class TestThreadState(unittest.TestCase): @@ -545,6 +547,7 @@ class TestThreadState(unittest.TestCase): t.start() t.join() + class Test_testcapi(unittest.TestCase): def test__testcapi(self): for name in dir(_testcapi): @@ -553,5 +556,61 @@ class Test_testcapi(unittest.TestCase): test = getattr(_testcapi, name) test() + +class MallocTests(unittest.TestCase): + ENV = 'debug' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure('-c', code, PYTHONMALLOC=self.ENV) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are not all FORBIDDENBYTE \(0x[0-9a-f]{2}\):\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfb\n" + r" at tail\+2: 0xfb\n" + r" at tail\+3: 0xfb\n" + r" at tail\+4: 0xfb\n" + r" at tail\+5: 0xfb\n" + r" at tail\+6: 0xfb\n" + r" at tail\+7: 0xfb\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" + r"Fatal Python error: bad trailing pad byte") + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are FORBIDDENBYTE, as expected.\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: .*\n" + r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") + self.assertRegex(out, regex) + + +class MallocDebugTests(MallocTests): + ENV = 'malloc_debug' + + +@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, + 'need pymalloc') +class PymallocDebugTests(MallocTests): + ENV = 'pymalloc_debug' + + +@unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG') +class DefaultMallocDebugTests(MallocTests): + ENV = '' + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 1c894a4..852be06 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,19 @@ Release date: tba Core and Builtins ----------------- +- Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the + Python memory allocators and/or install debug hooks. + +- Issue #26516: The :c:func`PyMem_SetupDebugHooks` function can now also be + used on Python compiled in release mode. + +- Issue #26516: The :envvar:`PYTHONMALLOCSTATS` environment variable can now + also be used on Python compiled in release mode. It now has no effect if + set to an empty string. + +- Issue #26516: In debug mode, debug hooks are now also installed on Python + memory allocators when Python is configured without pymalloc. + - Issue #26464: Fix str.translate() when string is ASCII and first replacements removes character, but next replacement uses a non-ASCII character or a string longer than 1 character. Regression introduced in Python 3.5.0. diff --git a/Misc/README.valgrind b/Misc/README.valgrind index b5a9a32..908f137 100644 --- a/Misc/README.valgrind +++ b/Misc/README.valgrind @@ -2,6 +2,9 @@ This document describes some caveats about the use of Valgrind with Python. Valgrind is used periodically by Python developers to try to ensure there are no memory leaks or invalid memory reads/writes. +UPDATE: Python 3.6 now supports PYTHONMALLOC=malloc environment variable which +can be used to force the usage of the malloc() allocator of the C library. + If you don't want to read about the details of using Valgrind, there are still two things you must do to suppress the warnings. First, you must use a suppressions file. One is supplied in diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a21a584..babffc4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3616,6 +3616,33 @@ get_recursion_depth(PyObject *self, PyObject *args) return PyLong_FromLong(tstate->recursion_depth - 1); } +static PyObject* +pymem_buffer_overflow(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate buffer overflow to check that PyMem_Free() detects + the overflow when debug hooks are installed. */ + buffer = PyMem_Malloc(16); + buffer[16] = 'x'; + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + +static PyObject* +pymem_api_misuse(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate misusage of Python allocators: + allococate with PyMem but release with PyMem_Raw. */ + buffer = PyMem_Malloc(16); + PyMem_RawFree(buffer); + + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3798,6 +3825,8 @@ static PyMethodDef TestMethods[] = { {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, + {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, + {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/main.c b/Modules/main.c index ee129a5..b6dcdd0 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -93,14 +93,15 @@ static const char usage_5[] = " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n\ -"; -static const char usage_6[] = "\ -PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n\ - to seed the hashes of str, bytes and datetime objects. It can also be\n\ - set to an integer in the range [0,4294967295] to get hash values with a\n\ - predictable seed.\n\ -"; +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n"; +static const char usage_6[] = +"PYTHONHASHSEED: if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str, bytes and datetime objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash values with a\n" +" predictable seed.\n" +"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n" +" hooks.\n"; static int usage(int exitcode, const wchar_t* program) @@ -341,6 +342,7 @@ Py_Main(int argc, wchar_t **argv) int help = 0; int version = 0; int saw_unbuffered_flag = 0; + char *opt; PyCompilerFlags cf; PyObject *warning_option = NULL; PyObject *warning_options = NULL; @@ -365,6 +367,13 @@ Py_Main(int argc, wchar_t **argv) } } + opt = Py_GETENV("PYTHONMALLOC"); + if (_PyMem_SetupAllocators(opt) < 0) { + fprintf(stderr, + "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt); + exit(1); + } + Py_HashRandomizationFlag = 1; _PyRandom_Init(); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7cc889f..e4bd8ac 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,7 +2,19 @@ /* Python's malloc wrappers (see pymem.h) */ -#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +/* + * Basic types + * I don't care if these are defined in or elsewhere. Axiom. + */ +#undef uchar +#define uchar unsigned char /* assuming == 8 bits */ + +#undef uint +#define uint unsigned int /* assuming >= 16 bits */ + +#undef uptr +#define uptr Py_uintptr_t + /* Forward declaration */ static void* _PyMem_DebugMalloc(void *ctx, size_t size); static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); @@ -11,7 +23,6 @@ static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); -#endif #if defined(__has_feature) /* Clang */ #if __has_feature(address_sanitizer) /* is ASAN enabled? */ @@ -147,7 +158,6 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size) #endif #define PYMEM_FUNCS PYRAW_FUNCS -#ifdef PYMALLOC_DEBUG typedef struct { /* We tag each block with an API ID in order to tag API violations */ char api_id; @@ -164,10 +174,9 @@ static struct { }; #define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree -#endif static PyMemAllocatorEx _PyMem_Raw = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.raw, PYDBG_FUNCS #else NULL, PYRAW_FUNCS @@ -175,7 +184,7 @@ static PyMemAllocatorEx _PyMem_Raw = { }; static PyMemAllocatorEx _PyMem = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.mem, PYDBG_FUNCS #else NULL, PYMEM_FUNCS @@ -183,13 +192,71 @@ static PyMemAllocatorEx _PyMem = { }; static PyMemAllocatorEx _PyObject = { -#ifdef PYMALLOC_DEBUG +#ifdef Py_DEBUG &_PyMem_Debug.obj, PYDBG_FUNCS #else NULL, PYOBJ_FUNCS #endif }; +int +_PyMem_SetupAllocators(const char *opt) +{ + if (opt == NULL || *opt == '\0') { + /* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line + options): use default allocators */ +#ifdef Py_DEBUG +# ifdef WITH_PYMALLOC + opt = "pymalloc_debug"; +# else + opt = "malloc_debug"; +# endif +#else + /* !Py_DEBUG */ +# ifdef WITH_PYMALLOC + opt = "pymalloc"; +# else + opt = "malloc"; +# endif +#endif + } + + if (strcmp(opt, "debug") == 0) { + PyMem_SetupDebugHooks(); + } + else if (strcmp(opt, "malloc") == 0 || strcmp(opt, "malloc_debug") == 0) + { + PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + if (strcmp(opt, "malloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#ifdef WITH_PYMALLOC + else if (strcmp(opt, "pymalloc") == 0 + || strcmp(opt, "pymalloc_debug") == 0) + { + PyMemAllocatorEx mem_alloc = {NULL, PYRAW_FUNCS}; + PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); + + if (strcmp(opt, "pymalloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#endif + else { + /* unknown allocator */ + return -1; + } + return 0; +} + #undef PYRAW_FUNCS #undef PYMEM_FUNCS #undef PYOBJ_FUNCS @@ -205,12 +272,34 @@ static PyObjectArenaAllocator _PyObject_Arena = {NULL, #endif }; +static int +_PyMem_DebugEnabled(void) +{ + return (_PyObject.malloc == _PyMem_DebugMalloc); +} + +#ifdef WITH_PYMALLOC +int +_PyMem_PymallocEnabled(void) +{ + if (_PyMem_DebugEnabled()) { + return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc); + } + else { + return (_PyObject.malloc == _PyObject_Malloc); + } +} +#endif + void PyMem_SetupDebugHooks(void) { -#ifdef PYMALLOC_DEBUG PyMemAllocatorEx alloc; + /* hooks already installed */ + if (_PyMem_DebugEnabled()) + return; + alloc.malloc = _PyMem_DebugMalloc; alloc.calloc = _PyMem_DebugCalloc; alloc.realloc = _PyMem_DebugRealloc; @@ -233,7 +322,6 @@ PyMem_SetupDebugHooks(void) PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } -#endif } void @@ -264,7 +352,6 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break; /* ignore unknown domain */ } - } void @@ -642,22 +729,6 @@ static int running_on_valgrind = -1; #define SIMPLELOCK_LOCK(lock) /* acquire released lock */ #define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ -/* - * Basic types - * I don't care if these are defined in or elsewhere. Axiom. - */ -#undef uchar -#define uchar unsigned char /* assuming == 8 bits */ - -#undef uint -#define uint unsigned int /* assuming >= 16 bits */ - -#undef ulong -#define ulong unsigned long /* assuming >= 32 bits */ - -#undef uptr -#define uptr Py_uintptr_t - /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; @@ -949,11 +1020,15 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; + static int debug_stats = -1; -#ifdef PYMALLOC_DEBUG - if (Py_GETENV("PYTHONMALLOCSTATS")) + if (debug_stats == -1) { + char *opt = Py_GETENV("PYTHONMALLOCSTATS"); + debug_stats = (opt != NULL && *opt != '\0'); + } + if (debug_stats) _PyObject_DebugMallocStats(stderr); -#endif + if (unused_arena_objects == NULL) { uint i; uint numarenas; @@ -1709,7 +1784,7 @@ _Py_GetAllocatedBlocks(void) #endif /* WITH_PYMALLOC */ -#ifdef PYMALLOC_DEBUG + /*==========================================================================*/ /* A x-platform debugging allocator. This doesn't manage memory directly, * it wraps a real allocator, adding extra debugging info to the memory blocks. @@ -1767,31 +1842,6 @@ write_size_t(void *p, size_t n) } } -#ifdef Py_DEBUG -/* Is target in the list? The list is traversed via the nextpool pointers. - * The list may be NULL-terminated, or circular. Return 1 if target is in - * list, else 0. - */ -static int -pool_is_in_list(const poolp target, poolp list) -{ - poolp origlist = list; - assert(target != NULL); - if (list == NULL) - return 0; - do { - if (target == list) - return 1; - list = list->nextpool; - } while (list != NULL && list != origlist); - return 0; -} - -#else -#define pool_is_in_list(X, Y) 1 - -#endif /* Py_DEBUG */ - /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2106,7 +2156,6 @@ _PyObject_DebugDumpAddress(const void *p) } } -#endif /* PYMALLOC_DEBUG */ static size_t printone(FILE *out, const char* msg, size_t value) @@ -2158,8 +2207,30 @@ _PyDebugAllocatorStats(FILE *out, (void)printone(out, buf2, num_blocks * sizeof_block); } + #ifdef WITH_PYMALLOC +#ifdef Py_DEBUG +/* Is target in the list? The list is traversed via the nextpool pointers. + * The list may be NULL-terminated, or circular. Return 1 if target is in + * list, else 0. + */ +static int +pool_is_in_list(const poolp target, poolp list) +{ + poolp origlist = list; + assert(target != NULL); + if (list == NULL) + return 0; + do { + if (target == list) + return 1; + list = list->nextpool; + } while (list != NULL && list != origlist); + return 0; +} +#endif + /* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. @@ -2233,7 +2304,9 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ +#ifdef Py_DEBUG assert(pool_is_in_list(p, arenas[i].freepools)); +#endif continue; } ++numpools[sz]; @@ -2273,9 +2346,8 @@ _PyObject_DebugMallocStats(FILE *out) quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); -#ifdef PYMALLOC_DEBUG - (void)printone(out, "# times object malloc called", serialno); -#endif + if (_PyMem_DebugEnabled()) + (void)printone(out, "# times object malloc called", serialno); (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater); @@ -2303,6 +2375,7 @@ _PyObject_DebugMallocStats(FILE *out) #endif /* #ifdef WITH_PYMALLOC */ + #ifdef Py_USING_MEMORY_DEBUGGER /* Make this function last so gcc won't inline it since the definition is * after the reference. diff --git a/Programs/python.c b/Programs/python.c index 37b10b8..a7afbc7 100644 --- a/Programs/python.c +++ b/Programs/python.c @@ -24,6 +24,9 @@ main(int argc, char **argv) int i, res; char *oldloc; + /* Force malloc() allocator to bootstrap Python */ + (void)_PyMem_SetupAllocators("malloc"); + argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); if (!argv_copy || !argv_copy2) { @@ -62,7 +65,13 @@ main(int argc, char **argv) setlocale(LC_ALL, oldloc); PyMem_RawFree(oldloc); + res = Py_Main(argc, argv_copy); + + /* Force again malloc() allocator to release memory blocks allocated + before Py_Main() */ + (void)_PyMem_SetupAllocators("malloc"); + for (i = 0; i < argc; i++) { PyMem_RawFree(argv_copy2[i]); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e9db7f6..715a547 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -702,9 +702,12 @@ Py_FinalizeEx(void) if (Py_GETENV("PYTHONDUMPREFS")) _Py_PrintReferenceAddresses(stderr); #endif /* Py_TRACE_REFS */ -#ifdef PYMALLOC_DEBUG - if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(stderr); +#ifdef WITH_PYMALLOC + if (_PyMem_PymallocEnabled()) { + char *opt = Py_GETENV("PYTHONMALLOCSTATS"); + if (opt != NULL && *opt != '\0') + _PyObject_DebugMallocStats(stderr); + } #endif call_ll_exitfuncs(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 702e8f0..c5b4ac1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1151,8 +1151,10 @@ static PyObject * sys_debugmallocstats(PyObject *self, PyObject *args) { #ifdef WITH_PYMALLOC - _PyObject_DebugMallocStats(stderr); - fputc('\n', stderr); + if (_PyMem_PymallocEnabled()) { + _PyObject_DebugMallocStats(stderr); + fputc('\n', stderr); + } #endif _PyObject_DebugTypeStats(stderr); -- cgit v0.12 From 791da1cc264574f8f3e44570d0fce293f755fdf3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 16:53:12 +0100 Subject: Fix Py_FatalError() if called without the GIL Issue #26558: If Py_FatalError() is called without the GIL, don't try to print the current exception, nor try to flush stdout and stderr: only dump the traceback of Python threads. --- Python/pylifecycle.c | 84 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 715a547..aaf5811 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1267,31 +1267,62 @@ initstdio(void) } +static void +_Py_FatalError_DumpTracebacks(int fd) +{ + PyThreadState *tstate; + +#ifdef WITH_THREAD + /* PyGILState_GetThisThreadState() works even if the GIL was released */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_GET(); +#endif + if (tstate == NULL) { + /* _Py_DumpTracebackThreads() requires the thread state to display + * frames */ + return; + } + + fputc('\n', stderr); + fflush(stderr); + + /* display the current Python stack */ + _Py_DumpTracebackThreads(fd, tstate->interp, tstate); +} + /* Print the current exception (if an exception is set) with its traceback, - * or display the current Python stack. - * - * Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is - * called on catastrophic cases. */ + or display the current Python stack. -static void -_Py_PrintFatalError(int fd) + Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is + called on catastrophic cases. + + Return 1 if the traceback was displayed, 0 otherwise. */ + +static int +_Py_FatalError_PrintExc(int fd) { PyObject *ferr, *res; PyObject *exception, *v, *tb; int has_tb; - PyThreadState *tstate; + + if (PyThreadState_GET() == NULL) { + /* The GIL is released: trying to acquire it is likely to deadlock, + just give up. */ + return 0; + } PyErr_Fetch(&exception, &v, &tb); if (exception == NULL) { /* No current exception */ - goto display_stack; + return 0; } ferr = _PySys_GetObjectId(&PyId_stderr); if (ferr == NULL || ferr == Py_None) { /* sys.stderr is not set yet or set to None, no need to try to display the exception */ - goto display_stack; + return 0; } PyErr_NormalizeException(&exception, &v, &tb); @@ -1302,7 +1333,7 @@ _Py_PrintFatalError(int fd) PyException_SetTraceback(v, tb); if (exception == NULL) { /* PyErr_NormalizeException() failed */ - goto display_stack; + return 0; } has_tb = (tb != Py_None); @@ -1318,28 +1349,9 @@ _Py_PrintFatalError(int fd) else Py_DECREF(res); - if (has_tb) - return; - -display_stack: -#ifdef WITH_THREAD - /* PyGILState_GetThisThreadState() works even if the GIL was released */ - tstate = PyGILState_GetThisThreadState(); -#else - tstate = PyThreadState_GET(); -#endif - if (tstate == NULL) { - /* _Py_DumpTracebackThreads() requires the thread state to display - * frames */ - return; - } - - fputc('\n', stderr); - fflush(stderr); - - /* display the current Python stack */ - _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + return has_tb; } + /* Print fatal error message and abort */ void @@ -1365,10 +1377,14 @@ Py_FatalError(const char *msg) /* Print the exception (if an exception is set) with its traceback, * or display the current Python stack. */ - _Py_PrintFatalError(fd); + if (!_Py_FatalError_PrintExc(fd)) + _Py_FatalError_DumpTracebacks(fd); - /* Flush sys.stdout and sys.stderr */ - flush_std_files(); + /* Check if the current Python thread hold the GIL */ + if (PyThreadState_GET() != NULL) { + /* Flush sys.stdout and sys.stderr */ + flush_std_files(); + } /* The main purpose of faulthandler is to display the traceback. We already * did our best to display it. So faulthandler can now be disabled. -- cgit v0.12 From d222653f8f39733f65ea93bfeab7748185ba4ea8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:01:32 +0100 Subject: Issue #26558: Remove useless check in tracemalloc The first instruction of tracemalloc_add_trace() is traceback_new() which already checks the GIL. --- Modules/_tracemalloc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 226a473..d62e682 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -439,10 +439,6 @@ tracemalloc_add_trace(void *ptr, size_t size) trace_t trace; int res; -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - traceback = traceback_new(); if (traceback == NULL) return -1; -- cgit v0.12 From a1bc28a91de9bfe88fc16cab400cd50dd1a27467 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:10:36 +0100 Subject: Issue #26516: Fix test_capi on Windows Pointers are formatted differently. --- Lib/test/test_capi.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index d56d702..6a066f1 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -559,6 +559,8 @@ class Test_testcapi(unittest.TestCase): class MallocTests(unittest.TestCase): ENV = 'debug' + # '0x04c06e0' or '04C06E0' + PTR_REGEX = r'(?:0x[0-9a-f]+|[0-9A-F]+)' def check(self, code): with support.SuppressCrashReport(): @@ -568,10 +570,10 @@ class MallocTests(unittest.TestCase): def test_buffer_overflow(self): out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') - regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail=0x[0-9a-f]+ are not all FORBIDDENBYTE \(0x[0-9a-f]{2}\):\n" + r" The 8 pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" r" at tail\+0: 0x78 \*\*\* OUCH\n" r" at tail\+1: 0xfb\n" r" at tail\+2: 0xfb\n" @@ -583,17 +585,19 @@ class MallocTests(unittest.TestCase): r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" r"Fatal Python error: bad trailing pad byte") + regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) def test_api_misuse(self): out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') - regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail=0x[0-9a-f]+ are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: .*\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") + regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From b3adb1adebc8090e8b838a7b921a4efef1adcf4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 17:40:09 +0100 Subject: Issue #26516: Fix test_capi on 32-bit system On 32-bit system, only 4 bytes after dumped for the tail. --- Lib/test/test_capi.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6a066f1..5761686 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -4,6 +4,7 @@ import os import pickle import random +import re import subprocess import sys import sysconfig @@ -572,30 +573,27 @@ class MallocTests(unittest.TestCase): out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" - r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" r" at tail\+0: 0x78 \*\*\* OUCH\n" r" at tail\+1: 0xfb\n" r" at tail\+2: 0xfb\n" - r" at tail\+3: 0xfb\n" - r" at tail\+4: 0xfb\n" - r" at tail\+5: 0xfb\n" - r" at tail\+6: 0xfb\n" - r" at tail\+7: 0xfb\n" + r" .*\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" + r" Data at p: cb cb cb .*\n" r"Fatal Python error: bad trailing pad byte") regex = regex.format(ptr=self.PTR_REGEX) + regex = re.compile(regex, flags=re.DOTALL) self.assertRegex(out, regex) def test_api_misuse(self): out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') regex = (r"Debug memory block at address p={ptr}: API 'm'\n" r" 16 bytes originally requested\n" - r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" - r" The 8 pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" + r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: .*\n" + r" Data at p: cb cb cb .*\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From c44f70770bf629469a5a179b643e53dfeca884ad Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 18:07:53 +0100 Subject: posix_getcwd(): limit to INT_MAX on Windows It's more to fix a conversion warning during compilation, I don't think that Windows support current working directory larger than 2 GB ... --- Modules/posixmodule.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6e0081d..7e89878 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3320,12 +3320,22 @@ posix_getcwd(int use_bytes) Py_BEGIN_ALLOW_THREADS do { buflen += chunk; +#ifdef MS_WINDOWS + if (buflen > INT_MAX) { + PyErr_NoMemory(); + break; + } +#endif tmpbuf = PyMem_RawRealloc(buf, buflen); if (tmpbuf == NULL) break; buf = tmpbuf; +#ifdef MS_WINDOWS + cwd = getcwd(buf, (int)buflen); +#else cwd = getcwd(buf, buflen); +#endif } while (cwd == NULL && errno == ERANGE); Py_END_ALLOW_THREADS -- cgit v0.12 From 21b47117ac061f0ba08219aa1a6d81a4a5a9c483 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 18:09:39 +0100 Subject: _pickle: Fix load_counted_tuple(), use Py_ssize_t for size Fix a warning on Windows 64-bit. --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index bb3330a..5c3530b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5056,7 +5056,7 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) } static int -load_counted_tuple(UnpicklerObject *self, int len) +load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) { PyObject *tuple; -- cgit v0.12 From 08572f68a980576e883e4eba1227b59782d5192e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 21:55:43 +0100 Subject: Issue #26516: Fix test_capi on AIX Fix regex for parse a pointer address. --- Lib/test/test_capi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 5761686..6c940ef 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -561,7 +561,7 @@ class Test_testcapi(unittest.TestCase): class MallocTests(unittest.TestCase): ENV = 'debug' # '0x04c06e0' or '04C06E0' - PTR_REGEX = r'(?:0x[0-9a-f]+|[0-9A-F]+)' + PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' def check(self, code): with support.SuppressCrashReport(): -- cgit v0.12 From 8a1be61849341528c866924cf69d378ce7790889 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 22:07:55 +0100 Subject: Add more checks on the GIL Issue #10915, #15751, #26558: * PyGILState_Check() now returns 1 (success) before the creation of the GIL and after the destruction of the GIL. It allows to use the function early in Python initialization and late in Python finalization. * Add a flag to disable PyGILState_Check(). Disable PyGILState_Check() when Py_NewInterpreter() is called * Add assert(PyGILState_Check()) to: _Py_dup(), _Py_fstat(), _Py_read() and _Py_write() --- Include/pystate.h | 4 ++++ Parser/pgenmain.c | 8 ++++++++ Python/fileutils.c | 16 ++++++++++++++++ Python/pylifecycle.c | 5 +++++ Python/pystate.c | 24 +++++++++++++++++++----- 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index b50b16b..550c332 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -247,6 +247,10 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); * currently holds the GIL, 0 otherwise */ #ifndef Py_LIMITED_API +/* Issue #26558: Flag to disable PyGILState_Check(). + If set, PyGILState_Check() always return 1. */ +PyAPI_DATA(int) _PyGILState_check_enabled; + PyAPI_FUNC(int) PyGILState_Check(void); #endif diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 3ca4afe..671a0cb 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -37,6 +37,14 @@ Py_Exit(int sts) exit(sts); } +#ifdef WITH_THREAD +/* Needed by obmalloc.c */ +int PyGILState_Check(void) +{ + return 1; +} +#endif + int main(int argc, char **argv) { diff --git a/Python/fileutils.c b/Python/fileutils.c index 06d632a..a710c99 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -683,6 +683,10 @@ _Py_fstat(int fd, struct _Py_stat_struct *status) { int res; +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + Py_BEGIN_ALLOW_THREADS res = _Py_fstat_noraise(fd, status); Py_END_ALLOW_THREADS @@ -1164,6 +1168,10 @@ _Py_read(int fd, void *buf, size_t count) int err; int async_err = 0; +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + /* _Py_read() must not be called with an exception set, otherwise the * caller may think that read() was interrupted by a signal and the signal * handler raised an exception. */ @@ -1319,6 +1327,10 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) Py_ssize_t _Py_write(int fd, const void *buf, size_t count) { +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + /* _Py_write() must not be called with an exception set, otherwise the * caller may think that write() was interrupted by a signal and the signal * handler raised an exception. */ @@ -1468,6 +1480,10 @@ _Py_dup(int fd) DWORD ftype; #endif +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (!_PyVerify_fd(fd)) { PyErr_SetFromErrno(PyExc_OSError); return -1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index aaf5811..4fc6a15 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -692,6 +692,7 @@ Py_FinalizeEx(void) /* Delete current thread. After this, many C API calls become crashy. */ PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); #ifdef Py_TRACE_REFS @@ -743,6 +744,10 @@ Py_NewInterpreter(void) if (!initialized) Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); + /* Issue #10915, #15751: The GIL API doesn't work with multiple + interpreters: disable PyGILState_Check(). */ + _PyGILState_check_enabled = 0; + interp = PyInterpreterState_New(); if (interp == NULL) return NULL; diff --git a/Python/pystate.c b/Python/pystate.c index 853e5c7..e8026c5 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -34,6 +34,8 @@ to avoid the expense of doing their own locking). extern "C" { #endif +int _PyGILState_check_enabled = 1; + #ifdef WITH_THREAD #include "pythread.h" static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */ @@ -45,7 +47,7 @@ static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */ GILState implementation */ static PyInterpreterState *autoInterpreterState = NULL; -static int autoTLSkey = 0; +static int autoTLSkey = -1; #else #define HEAD_INIT() /* Nothing */ #define HEAD_LOCK() /* Nothing */ @@ -449,10 +451,10 @@ PyThreadState_DeleteCurrent() if (tstate == NULL) Py_FatalError( "PyThreadState_DeleteCurrent: no current tstate"); - SET_TSTATE(NULL); + tstate_delete_common(tstate); if (autoInterpreterState && PyThread_get_key_value(autoTLSkey) == tstate) PyThread_delete_key_value(autoTLSkey); - tstate_delete_common(tstate); + SET_TSTATE(NULL); PyEval_ReleaseLock(); } #endif /* WITH_THREAD */ @@ -716,6 +718,7 @@ void _PyGILState_Fini(void) { PyThread_delete_key(autoTLSkey); + autoTLSkey = -1; autoInterpreterState = NULL; } @@ -784,8 +787,19 @@ PyGILState_GetThisThreadState(void) int PyGILState_Check(void) { - PyThreadState *tstate = GET_TSTATE(); - return tstate && (tstate == PyGILState_GetThisThreadState()); + PyThreadState *tstate; + + if (!_PyGILState_check_enabled) + return 1; + + if (autoTLSkey == -1) + return 1; + + tstate = GET_TSTATE(); + if (tstate == NULL) + return 0; + + return (tstate == PyGILState_GetThisThreadState()); } PyGILState_STATE -- cgit v0.12 From c4aec3628b6effcf205d70ce7d80f69847954b66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2016 22:26:53 +0100 Subject: Check the GIL in PyObject_Malloc() Issue #26558: The debug hook of PyObject_Malloc() now checks that the GIL is held when the function is called. --- Doc/c-api/memory.rst | 9 +++-- Doc/whatsnew/3.6.rst | 3 ++ Lib/test/test_capi.py | 30 +++++++++++----- Misc/NEWS | 4 +++ Modules/_testcapimodule.c | 15 ++++++++ Objects/obmalloc.c | 92 +++++++++++++++++++++++++++++++++++------------ 6 files changed, 119 insertions(+), 34 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index fe1cd5f..25867d9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -341,10 +341,13 @@ Customize Memory Allocators Newly allocated memory is filled with the byte ``0xCB``, freed memory is filled with the byte ``0xDB``. Additional checks: - - detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer allocated by :c:func:`PyMem_Malloc` - - detect write before the start of the buffer (buffer underflow) - - detect write after the end of the buffer (buffer overflow) + - Detect write before the start of the buffer (buffer underflow) + - Detect write after the end of the buffer (buffer overflow) + - Check that the :term:`GIL ` is held when + allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: + :c:func:`PyObject_Malloc`) are called These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 588826b..b644a5c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -117,6 +117,9 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: :c:func:`PyMem_Malloc`. * Detect write before the start of the buffer (buffer underflow) * Detect write after the end of the buffer (buffer overflow) +* Check that the :term:`GIL ` is held when allocator + functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: + :c:func:`PyObject_Malloc`) are called See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 6c940ef..1de19a0 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -441,6 +441,7 @@ class EmbeddingTests(unittest.TestCase): self.maxDiff = None self.assertEqual(out.strip(), expected_output) + class SkipitemTest(unittest.TestCase): def test_skipitem(self): @@ -558,14 +559,15 @@ class Test_testcapi(unittest.TestCase): test() -class MallocTests(unittest.TestCase): - ENV = 'debug' +class PyMemDebugTests(unittest.TestCase): + PYTHONMALLOC = 'debug' # '0x04c06e0' or '04C06E0' PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' def check(self, code): with support.SuppressCrashReport(): - out = assert_python_failure('-c', code, PYTHONMALLOC=self.ENV) + out = assert_python_failure('-c', code, + PYTHONMALLOC=self.PYTHONMALLOC) stderr = out.err return stderr.decode('ascii', 'replace') @@ -598,20 +600,30 @@ class MallocTests(unittest.TestCase): regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) + def test_pyobject_malloc_without_gil(self): + # Calling PyObject_Malloc() without holding the GIL must raise an + # error in debug mode. + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + out = self.check(code) + expected = ('Fatal Python error: Python memory allocator called ' + 'without holding the GIL') + self.assertIn(expected, out) + -class MallocDebugTests(MallocTests): - ENV = 'malloc_debug' +class PyMemMallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'malloc_debug' @unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, 'need pymalloc') -class PymallocDebugTests(MallocTests): - ENV = 'pymalloc_debug' +class PyMemPymallocDebugTests(PyMemDebugTests): + PYTHONMALLOC = 'pymalloc_debug' @unittest.skipUnless(Py_DEBUG, 'need Py_DEBUG') -class DefaultMallocDebugTests(MallocTests): - ENV = '' +class PyMemDefaultTests(PyMemDebugTests): + # test default allocator of Python compiled in debug mode + PYTHONMALLOC = '' if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 852be06..5233a43 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26558: The debug hooks on Python memory allocator + :c:func:`PyObject_Malloc` now detect when functions are called without + holding the GIL. + - Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the Python memory allocators and/or install debug hooks. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index babffc4..b3d8818 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3643,6 +3643,20 @@ pymem_api_misuse(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject* +pyobject_malloc_without_gil(PyObject *self, PyObject *args) +{ + char *buffer; + + Py_BEGIN_ALLOW_THREADS + buffer = PyObject_Malloc(10); + Py_END_ALLOW_THREADS + + PyObject_Free(buffer); + + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3827,6 +3841,7 @@ static PyMethodDef TestMethods[] = { {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, + {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e4bd8ac..f526b1f 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -16,10 +16,15 @@ #define uptr Py_uintptr_t /* Forward declaration */ +static void* _PyMem_DebugRawMalloc(void *ctx, size_t size); +static void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize); +static void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size); +static void _PyMem_DebugRawFree(void *ctx, void *p); + static void* _PyMem_DebugMalloc(void *ctx, size_t size); static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); -static void _PyMem_DebugFree(void *ctx, void *p); static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); +static void _PyMem_DebugFree(void *ctx, void *p); static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); @@ -173,11 +178,14 @@ static struct { {'o', {NULL, PYOBJ_FUNCS}} }; -#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree +#define PYRAWDBG_FUNCS \ + _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree +#define PYDBG_FUNCS \ + _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree static PyMemAllocatorEx _PyMem_Raw = { #ifdef Py_DEBUG - &_PyMem_Debug.raw, PYDBG_FUNCS + &_PyMem_Debug.raw, PYRAWDBG_FUNCS #else NULL, PYRAW_FUNCS #endif @@ -185,7 +193,7 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS + &_PyMem_Debug.mem, PYRAWDBG_FUNCS #else NULL, PYMEM_FUNCS #endif @@ -260,6 +268,7 @@ _PyMem_SetupAllocators(const char *opt) #undef PYRAW_FUNCS #undef PYMEM_FUNCS #undef PYOBJ_FUNCS +#undef PYRAWDBG_FUNCS #undef PYDBG_FUNCS static PyObjectArenaAllocator _PyObject_Arena = {NULL, @@ -296,27 +305,28 @@ PyMem_SetupDebugHooks(void) { PyMemAllocatorEx alloc; - /* hooks already installed */ - if (_PyMem_DebugEnabled()) - return; - - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc.malloc = _PyMem_DebugRawMalloc; + alloc.calloc = _PyMem_DebugRawCalloc; + alloc.realloc = _PyMem_DebugRawRealloc; + alloc.free = _PyMem_DebugRawFree; - if (_PyMem_Raw.malloc != _PyMem_DebugMalloc) { + if (_PyMem_Raw.malloc != _PyMem_DebugRawMalloc) { alloc.ctx = &_PyMem_Debug.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } - if (_PyMem.malloc != _PyMem_DebugMalloc) { + if (_PyMem.malloc != _PyMem_DebugRawMalloc) { alloc.ctx = &_PyMem_Debug.mem; PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); } + alloc.malloc = _PyMem_DebugMalloc; + alloc.calloc = _PyMem_DebugCalloc; + alloc.realloc = _PyMem_DebugRealloc; + alloc.free = _PyMem_DebugFree; + if (_PyObject.malloc != _PyMem_DebugMalloc) { alloc.ctx = &_PyMem_Debug.obj; PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); @@ -1869,7 +1879,7 @@ p[2*S+n+S: 2*S+n+2*S] */ static void * -_PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes) +_PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *p; /* base address of malloc'ed block */ @@ -1906,18 +1916,18 @@ _PyMem_DebugAlloc(int use_calloc, void *ctx, size_t nbytes) } static void * -_PyMem_DebugMalloc(void *ctx, size_t nbytes) +_PyMem_DebugRawMalloc(void *ctx, size_t nbytes) { - return _PyMem_DebugAlloc(0, ctx, nbytes); + return _PyMem_DebugRawAlloc(0, ctx, nbytes); } static void * -_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) +_PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize) { size_t nbytes; assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; - return _PyMem_DebugAlloc(1, ctx, nbytes); + return _PyMem_DebugRawAlloc(1, ctx, nbytes); } /* The debug free first checks the 2*SST bytes on each end for sanity (in @@ -1926,7 +1936,7 @@ _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) Then calls the underlying free. */ static void -_PyMem_DebugFree(void *ctx, void *p) +_PyMem_DebugRawFree(void *ctx, void *p) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */ @@ -1943,7 +1953,7 @@ _PyMem_DebugFree(void *ctx, void *p) } static void * -_PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) +_PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) { debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *q = (uchar *)p, *oldq; @@ -1953,7 +1963,7 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) int i; if (p == NULL) - return _PyMem_DebugAlloc(0, ctx, nbytes); + return _PyMem_DebugRawAlloc(0, ctx, nbytes); _PyMem_DebugCheckAddress(api->api_id, p); bumpserialno(); @@ -1996,6 +2006,44 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes) return q; } +static void +_PyMem_DebugCheckGIL(void) +{ +#ifdef WITH_THREAD + if (!PyGILState_Check()) + Py_FatalError("Python memory allocator called " + "without holding the GIL"); +#endif +} + +static void * +_PyMem_DebugMalloc(void *ctx, size_t nbytes) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawMalloc(ctx, nbytes); +} + +static void * +_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawCalloc(ctx, nelem, elsize); +} + +static void +_PyMem_DebugFree(void *ctx, void *ptr) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawFree(ctx, ptr); +} + +static void * +_PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) +{ + _PyMem_DebugCheckGIL(); + return _PyMem_DebugRawRealloc(ctx, ptr, nbytes); +} + /* Check the forbidden bytes on both ends of the memory allocated for p. * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress, * and call Py_FatalError to kill the program. -- cgit v0.12 From 32eb840a42ec0e131daac48d43aa35290e72571e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 11:12:35 +0100 Subject: Issue #26566: Rewrite test_signal.InterProcessSignalTests * Add Lib/test/signalinterproctester.py * Don't disable the garbage collector anymore * Don't use os.fork() with a subprocess to not inherit existing signal handlers or threads: start from a fresh process * Don't use UNIX kill command to send a signal but Python os.kill() * Use a timeout of 10 seconds to wait for the signal instead of 1 second * Always use signal.pause(), instead of time.wait(1), to wait for a signal * Use context manager on subprocess.Popen * remove code to retry on EINTR: it's no more needed since the PEP 475 * remove unused function exit_subprocess() * Cleanup the code --- Lib/test/signalinterproctester.py | 84 +++++++++++++++++++ Lib/test/test_signal.py | 171 ++------------------------------------ 2 files changed, 93 insertions(+), 162 deletions(-) create mode 100644 Lib/test/signalinterproctester.py diff --git a/Lib/test/signalinterproctester.py b/Lib/test/signalinterproctester.py new file mode 100644 index 0000000..d3ae170 --- /dev/null +++ b/Lib/test/signalinterproctester.py @@ -0,0 +1,84 @@ +import os +import signal +import subprocess +import sys +import time +import unittest + + +class SIGUSR1Exception(Exception): + pass + + +class InterProcessSignalTests(unittest.TestCase): + def setUp(self): + self.got_signals = {'SIGHUP': 0, 'SIGUSR1': 0, 'SIGALRM': 0} + + def sighup_handler(self, signum, frame): + self.got_signals['SIGHUP'] += 1 + + def sigusr1_handler(self, signum, frame): + self.got_signals['SIGUSR1'] += 1 + raise SIGUSR1Exception + + def wait_signal(self, child, signame, exc_class=None): + try: + if child is not None: + # This wait should be interrupted by exc_class + # (if set) + child.wait() + + timeout = 10.0 + deadline = time.monotonic() + timeout + + while time.monotonic() < deadline: + if self.got_signals[signame]: + return + signal.pause() + except BaseException as exc: + if exc_class is not None and isinstance(exc, exc_class): + # got the expected exception + return + raise + + self.fail('signal %s not received after %s seconds' + % (signame, timeout)) + + def subprocess_send_signal(self, pid, signame): + code = 'import os, signal; os.kill(%s, signal.%s)' % (pid, signame) + args = [sys.executable, '-I', '-c', code] + return subprocess.Popen(args) + + def test_interprocess_signal(self): + # Install handlers. This function runs in a sub-process, so we + # don't worry about re-setting the default handlers. + signal.signal(signal.SIGHUP, self.sighup_handler) + signal.signal(signal.SIGUSR1, self.sigusr1_handler) + signal.signal(signal.SIGUSR2, signal.SIG_IGN) + signal.signal(signal.SIGALRM, signal.default_int_handler) + + # Let the sub-processes know who to send signals to. + pid = str(os.getpid()) + + with self.subprocess_send_signal(pid, "SIGHUP") as child: + self.wait_signal(child, 'SIGHUP') + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 0, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR1") as child: + self.wait_signal(child, 'SIGUSR1', SIGUSR1Exception) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + with self.subprocess_send_signal(pid, "SIGUSR2") as child: + # Nothing should happen: SIGUSR2 is ignored + child.wait() + + signal.alarm(1) + self.wait_signal(None, 'SIGALRM', KeyboardInterrupt) + self.assertEqual(self.got_signals, {'SIGHUP': 1, 'SIGUSR1': 1, + 'SIGALRM': 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 1b80ff0..ab42ed7 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -22,29 +22,6 @@ except ImportError: _testcapi = None -class HandlerBCalled(Exception): - pass - - -def exit_subprocess(): - """Use os._exit(0) to exit the current subprocess. - - Otherwise, the test catches the SystemExit and continues executing - in parallel with the original test, so you wind up with an - exponential number of tests running concurrently. - """ - os._exit(0) - - -def ignoring_eintr(__func, *args, **kwargs): - try: - return __func(*args, **kwargs) - except OSError as e: - if e.errno != errno.EINTR: - raise - return None - - class GenericTests(unittest.TestCase): @unittest.skipIf(threading is None, "test needs threading module") @@ -63,145 +40,6 @@ class GenericTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") -class InterProcessSignalTests(unittest.TestCase): - MAX_DURATION = 20 # Entire test should last at most 20 sec. - - def setUp(self): - self.using_gc = gc.isenabled() - gc.disable() - - def tearDown(self): - if self.using_gc: - gc.enable() - - def format_frame(self, frame, limit=None): - return ''.join(traceback.format_stack(frame, limit=limit)) - - def handlerA(self, signum, frame): - self.a_called = True - - def handlerB(self, signum, frame): - self.b_called = True - raise HandlerBCalled(signum, self.format_frame(frame)) - - def wait(self, child): - """Wait for child to finish, ignoring EINTR.""" - while True: - try: - child.wait() - return - except OSError as e: - if e.errno != errno.EINTR: - raise - - def run_test(self): - # Install handlers. This function runs in a sub-process, so we - # don't worry about re-setting the default handlers. - signal.signal(signal.SIGHUP, self.handlerA) - signal.signal(signal.SIGUSR1, self.handlerB) - signal.signal(signal.SIGUSR2, signal.SIG_IGN) - signal.signal(signal.SIGALRM, signal.default_int_handler) - - # Variables the signals will modify: - self.a_called = False - self.b_called = False - - # Let the sub-processes know who to send signals to. - pid = os.getpid() - - child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) - if child: - self.wait(child) - if not self.a_called: - time.sleep(1) # Give the signal time to be delivered. - self.assertTrue(self.a_called) - self.assertFalse(self.b_called) - self.a_called = False - - # Make sure the signal isn't delivered while the previous - # Popen object is being destroyed, because __del__ swallows - # exceptions. - del child - try: - child = subprocess.Popen(['kill', '-USR1', str(pid)]) - # This wait should be interrupted by the signal's exception. - self.wait(child) - time.sleep(1) # Give the signal time to be delivered. - self.fail('HandlerBCalled exception not raised') - except HandlerBCalled: - self.assertTrue(self.b_called) - self.assertFalse(self.a_called) - - child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) - if child: - self.wait(child) # Nothing should happen. - - try: - signal.alarm(1) - # The race condition in pause doesn't matter in this case, - # since alarm is going to raise a KeyboardException, which - # will skip the call. - signal.pause() - # But if another signal arrives before the alarm, pause - # may return early. - time.sleep(1) - except KeyboardInterrupt: - pass - except: - self.fail("Some other exception woke us from pause: %s" % - traceback.format_exc()) - else: - self.fail("pause returned of its own accord, and the signal" - " didn't arrive after another second.") - - # Issue 3864, unknown if this affects earlier versions of freebsd also - @unittest.skipIf(sys.platform=='freebsd6', - 'inter process signals not reliable (do not mix well with threading) ' - 'on freebsd6') - def test_main(self): - # This function spawns a child process to insulate the main - # test-running process from all the signals. It then - # communicates with that child process over a pipe and - # re-raises information about any exceptions the child - # raises. The real work happens in self.run_test(). - os_done_r, os_done_w = os.pipe() - with closing(os.fdopen(os_done_r, 'rb')) as done_r, \ - closing(os.fdopen(os_done_w, 'wb')) as done_w: - child = os.fork() - if child == 0: - # In the child process; run the test and report results - # through the pipe. - try: - done_r.close() - # Have to close done_w again here because - # exit_subprocess() will skip the enclosing with block. - with closing(done_w): - try: - self.run_test() - except: - pickle.dump(traceback.format_exc(), done_w) - else: - pickle.dump(None, done_w) - except: - print('Uh oh, raised from pickle.') - traceback.print_exc() - finally: - exit_subprocess() - - done_w.close() - # Block for up to MAX_DURATION seconds for the test to finish. - r, w, x = select.select([done_r], [], [], self.MAX_DURATION) - if done_r in r: - tb = pickle.load(done_r) - if tb: - self.fail(tb) - else: - os.kill(child, signal.SIGKILL) - self.fail('Test deadlocked after %d seconds.' % - self.MAX_DURATION) - - -@unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass @@ -224,6 +62,15 @@ class PosixTests(unittest.TestCase): signal.signal(signal.SIGHUP, hup) self.assertEqual(signal.getsignal(signal.SIGHUP), hup) + # Issue 3864, unknown if this affects earlier versions of freebsd also + @unittest.skipIf(sys.platform=='freebsd6', + 'inter process signals not reliable (do not mix well with threading) ' + 'on freebsd6') + def test_interprocess_signal(self): + dirname = os.path.dirname(__file__) + script = os.path.join(dirname, 'signalinterproctester.py') + assert_python_ok(script) + @unittest.skipUnless(sys.platform == "win32", "Windows specific") class WindowsSignalTests(unittest.TestCase): -- cgit v0.12 From 7105e9f3de801ce50e24fcdfbdd450858e28a0fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 14:28:04 +0100 Subject: _tracemalloc: filename cannot be NULL --- Modules/_tracemalloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d62e682..2eb4dca 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -65,6 +65,8 @@ __attribute__((packed)) _declspec(align(4)) #endif { + /* filename cannot be NULL: "" is used if the Python frame + filename is NULL */ PyObject *filename; int lineno; } frame_t; @@ -987,8 +989,6 @@ frame_to_pyobject(frame_t *frame) if (frame_obj == NULL) return NULL; - if (frame->filename == NULL) - frame->filename = Py_None; Py_INCREF(frame->filename); PyTuple_SET_ITEM(frame_obj, 0, frame->filename); -- cgit v0.12 From 89e7cdcb9c124a01c5c3a69e086a0e8b26e65545 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 21:49:37 +0100 Subject: Enhance and rewrite traceback dump C functions Issue #26564: * Expose _Py_DumpASCII() and _Py_DumpDecimal() in traceback.h * Change the type of the second _Py_DumpASCII() parameter from int to unsigned long * Rewrite _Py_DumpDecimal() and dump_hexadecimal() to write directly characters in the expected order, avoid the need of reversing the string. * dump_hexadecimal() limits width to the size of the buffer * _Py_DumpASCII() does nothing if the object is not a Unicode string * dump_frame() wrtites "???" as the line number if the line number is negative --- Include/traceback.h | 18 +++++++++ Python/traceback.c | 109 +++++++++++++++++++++++++++------------------------- 2 files changed, 74 insertions(+), 53 deletions(-) diff --git a/Include/traceback.h b/Include/traceback.h index 891000c..7c630e3 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -67,6 +67,24 @@ PyAPI_DATA(const char*) _Py_DumpTracebackThreads( PyThreadState *current_thread); +#ifndef Py_LIMITED_API + +/* Write a Unicode object into the file descriptor fd. Encode the string to + ASCII using the backslashreplace error handler. + + Do nothing if text is not a Unicode object. The function accepts Unicode + string which is not ready (PyUnicode_WCHAR_KIND). + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); + +/* Format an integer as decimal into the file descriptor fd. + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpDecimal(int fd, unsigned long value); + +#endif /* !Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Python/traceback.c b/Python/traceback.c index 941d1cb..a40dbd1 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -479,40 +479,26 @@ PyTraceBack_Print(PyObject *v, PyObject *f) This function is signal safe. */ -static void -reverse_string(char *text, const size_t len) -{ - char tmp; - size_t i, j; - if (len == 0) - return; - for (i=0, j=len-1; i < j; i++, j--) { - tmp = text[i]; - text[i] = text[j]; - text[j] = tmp; - } -} - -/* Format an integer in range [0; 999999] to decimal, - and write it into the file fd. - - This function is signal safe. */ - -static void -dump_decimal(int fd, int value) +void +_Py_DumpDecimal(int fd, unsigned long value) { - char buffer[7]; - int len; - if (value < 0 || 999999 < value) - return; - len = 0; + /* maximum number of characters required for output of %lld or %p. + We need at most ceil(log10(256)*SIZEOF_LONG_LONG) digits, + plus 1 for the null byte. 53/22 is an upper bound for log10(256). */ + char buffer[1 + (sizeof(unsigned long)*53-1) / 22 + 1]; + char *ptr, *end; + + end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + ptr = end; + *ptr = '\0'; do { - buffer[len] = '0' + (value % 10); + --ptr; + assert(ptr >= buffer); + *ptr = '0' + (value % 10); value /= 10; - len++; } while (value); - reverse_string(buffer, len); - _Py_write_noraise(fd, buffer, len); + + _Py_write_noraise(fd, ptr, end - ptr); } /* Format an integer in range [0; 0xffffffff] to hexadecimal of 'width' digits, @@ -521,26 +507,29 @@ dump_decimal(int fd, int value) This function is signal safe. */ static void -dump_hexadecimal(int fd, unsigned long value, int width) +dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) { - int len; - char buffer[sizeof(unsigned long) * 2 + 1]; - len = 0; + Py_ssize_t size = sizeof(unsigned long) * 2; + char buffer[size + 1], *ptr, *end; + + if (width > size) + width = size; + + end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + ptr = end; + *ptr = '\0'; do { - buffer[len] = Py_hexdigits[value & 15]; + --ptr; + assert(ptr >= buffer); + *ptr = Py_hexdigits[value & 15]; value >>= 4; - len++; - } while (len < width || value); - reverse_string(buffer, len); - _Py_write_noraise(fd, buffer, len); -} - -/* Write an unicode object into the file fd using ascii+backslashreplace. + } while ((end - ptr) < width || value); - This function is signal safe. */ + _Py_write_noraise(fd, ptr, end - ptr); +} -static void -dump_ascii(int fd, PyObject *text) +void +_Py_DumpASCII(int fd, PyObject *text) { PyASCIIObject *ascii = (PyASCIIObject *)text; Py_ssize_t i, size; @@ -550,6 +539,9 @@ dump_ascii(int fd, PyObject *text) wchar_t *wstr = NULL; Py_UCS4 ch; + if (!PyUnicode_Check(text)) + return; + size = ascii->length; kind = ascii->state.kind; if (ascii->state.compact) { @@ -574,8 +566,9 @@ dump_ascii(int fd, PyObject *text) size = MAX_STRING_LENGTH; truncated = 1; } - else + else { truncated = 0; + } for (i=0; i < size; i++) { if (kind != PyUnicode_WCHAR_KIND) @@ -600,8 +593,9 @@ dump_ascii(int fd, PyObject *text) dump_hexadecimal(fd, ch, 8); } } - if (truncated) + if (truncated) { PUTS(fd, "..."); + } } /* Write a frame into the file fd: "File "xxx", line xxx in xxx". @@ -620,7 +614,7 @@ dump_frame(int fd, PyFrameObject *frame) && PyUnicode_Check(code->co_filename)) { PUTS(fd, "\""); - dump_ascii(fd, code->co_filename); + _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); } else { PUTS(fd, "???"); @@ -629,14 +623,21 @@ dump_frame(int fd, PyFrameObject *frame) /* PyFrame_GetLineNumber() was introduced in Python 2.7.0 and 3.2.0 */ lineno = PyCode_Addr2Line(code, frame->f_lasti); PUTS(fd, ", line "); - dump_decimal(fd, lineno); + if (lineno >= 0) { + _Py_DumpDecimal(fd, (unsigned long)lineno); + } + else { + PUTS(fd, "???"); + } PUTS(fd, " in "); if (code != NULL && code->co_name != NULL - && PyUnicode_Check(code->co_name)) - dump_ascii(fd, code->co_name); - else + && PyUnicode_Check(code->co_name)) { + _Py_DumpASCII(fd, code->co_name); + } + else { PUTS(fd, "???"); + } PUTS(fd, "\n"); } @@ -692,7 +693,9 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) PUTS(fd, "Current thread 0x"); else PUTS(fd, "Thread 0x"); - dump_hexadecimal(fd, (unsigned long)tstate->thread_id, sizeof(unsigned long)*2); + dump_hexadecimal(fd, + (unsigned long)tstate->thread_id, + sizeof(unsigned long) * 2); PUTS(fd, " (most recent call first):\n"); } -- cgit v0.12 From 0611c26a58a937dace420691e797fbea21db619b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 22:22:13 +0100 Subject: On memory error, dump the memory block traceback Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a memory block was allocated on memory block. Use the tracemalloc module to get the traceback. --- Doc/c-api/memory.rst | 7 +++++ Doc/whatsnew/3.6.rst | 43 +++++++++++++++++++++++++++++- Misc/NEWS | 4 +++ Modules/_tracemalloc.c | 67 +++++++++++++++++++++++++++++++++++++++-------- Modules/hashtable.c | 6 ++--- Objects/bytearrayobject.c | 1 + Objects/obmalloc.c | 9 +++++++ Parser/pgenmain.c | 8 +++--- 8 files changed, 126 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 25867d9..843ccac 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -349,12 +349,19 @@ Customize Memory Allocators allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: :c:func:`PyObject_Malloc`) are called + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. .. _pymalloc: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b644a5c..443e46a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -129,7 +129,48 @@ the C library for all Python memory allocations using ``PYTHONMALLOC=malloc``. It helps to use external memory debuggers like Valgrind on a Python compiled in release mode. -(Contributed by Victor Stinner in :issue:`26516`.) +On error, the debug hooks on Python memory allocators now use the +:mod:`tracemalloc` module to get the traceback where a memory block was +allocated. + +Example of fatal error on buffer overflow using +``python3.6 -X tracemalloc=5`` (store 5 frames in traces):: + + Debug memory block at address p=0x7fbcd41666f8: API 'o' + 4 bytes originally requested + The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected. + The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb): + at tail+0: 0x02 *** OUCH + at tail+1: 0xfb + at tail+2: 0xfb + at tail+3: 0xfb + at tail+4: 0xfb + at tail+5: 0xfb + at tail+6: 0xfb + at tail+7: 0xfb + The block was made by call #1233329 to debug malloc/realloc. + Data at p: 1a 2b 30 00 + + Memory block allocated at (most recent call first): + File "test/test_bytes.py", line 323 + File "unittest/case.py", line 600 + File "unittest/case.py", line 648 + File "unittest/suite.py", line 122 + File "unittest/suite.py", line 84 + + Fatal Python error: bad trailing pad byte + + Current thread 0x00007fbcdbd32700 (most recent call first): + File "test/test_bytes.py", line 323 in test_hex + File "unittest/case.py", line 600 in run + File "unittest/case.py", line 648 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + ... + +(Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.) Other Language Changes diff --git a/Misc/NEWS b/Misc/NEWS index 45769ba..cbb1a51 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26564: On error, the debug hooks on Python memory allocators now use + the :mod:`tracemalloc` module to get the traceback where a memory block was + allocated. + - Issue #26558: The debug hooks on Python memory allocator :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 1940af4..5752904 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1161,6 +1161,25 @@ finally: return get_traces.list; } +static traceback_t* +tracemalloc_get_traceback(const void *ptr) +{ + trace_t trace; + int found; + + if (!tracemalloc_config.tracing) + return NULL; + + TABLES_LOCK(); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (!found) + return NULL; + + return trace.traceback; +} + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1175,11 +1194,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) { PyTypeObject *type; void *ptr; - trace_t trace; - int found; - - if (!tracemalloc_config.tracing) - Py_RETURN_NONE; + traceback_t *traceback; type = Py_TYPE(obj); if (PyType_IS_GC(type)) @@ -1187,16 +1202,46 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); - TABLES_UNLOCK(); - - if (!found) + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) Py_RETURN_NONE; - return traceback_to_pyobject(trace.traceback, NULL); + return traceback_to_pyobject(traceback, NULL); +} + +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + +static void +_PyMem_DumpFrame(int fd, frame_t * frame) +{ + PUTS(fd, " File \""); + _Py_DumpASCII(fd, frame->filename); + PUTS(fd, "\", line "); + _Py_DumpDecimal(fd, frame->lineno); + PUTS(fd, "\n"); } +/* Dump the traceback where a memory block was allocated into file descriptor + fd. The function may block on TABLES_LOCK() but it is unlikely. */ +void +_PyMem_DumpTraceback(int fd, const void *ptr) +{ + traceback_t *traceback; + int i; + + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) + return; + + PUTS(fd, "Memory block allocated at (most recent call first):\n"); + for (i=0; i < traceback->nframe; i++) { + _PyMem_DumpFrame(fd, &traceback->frames[i]); + } + PUTS(fd, "\n"); +} + +#undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 133f313..7de154b 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -486,9 +486,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src) void *data, *new_data; dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func, &src->alloc); if (dst == NULL) return NULL; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 9e8ba39..67ae7d9 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,6 +2820,7 @@ bytearray_hex(PyBytesObject *self) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); + PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f526b1f..8812f59 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,10 @@ #include "Python.h" + +/* Defined in tracemalloc.c */ +extern void _PyMem_DumpTraceback(int fd, const void *ptr); + + /* Python's malloc wrappers (see pymem.h) */ /* @@ -2202,6 +2207,10 @@ _PyObject_DebugDumpAddress(const void *p) } fputc('\n', stderr); } + fputc('\n', stderr); + + fflush(stderr); + _PyMem_DumpTraceback(fileno(stderr), p); } diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 671a0cb..d5a13fe 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -38,11 +38,11 @@ Py_Exit(int sts) } #ifdef WITH_THREAD -/* Needed by obmalloc.c */ +/* Functions needed by obmalloc.c */ int PyGILState_Check(void) -{ - return 1; -} +{ return 1; } +void _PyMem_DumpTraceback(int fd, const void *ptr) +{} #endif int -- cgit v0.12 From ffcf1a54d39fcb2b9f9830b64708c423c2c4a18d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 22:49:40 +0100 Subject: Oops, revert unwanted change used to create an example Issue #26564. --- Objects/bytearrayobject.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 67ae7d9..9e8ba39 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,7 +2820,6 @@ bytearray_hex(PyBytesObject *self) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); - PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } -- cgit v0.12 From 6453e9ed0add5f60d150692cef4596702dcc3290 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 23:36:28 +0100 Subject: Issue #26564: Fix test_capi --- Lib/test/test_capi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 1de19a0..8e6245b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -583,6 +583,7 @@ class PyMemDebugTests(unittest.TestCase): r" .*\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb .*\n" + r"\n" r"Fatal Python error: bad trailing pad byte") regex = regex.format(ptr=self.PTR_REGEX) regex = re.compile(regex, flags=re.DOTALL) @@ -596,6 +597,7 @@ class PyMemDebugTests(unittest.TestCase): r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" r" Data at p: cb cb cb .*\n" + r"\n" r"Fatal Python error: bad ID: Allocated using API 'm', verified using API 'r'\n") regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) -- cgit v0.12 From 82f04e2dfdd2bdfef5bd94f8d6615ac0a1f620b4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 23:08:44 +0100 Subject: regrtest: Fix module.__path__ Issue #26538: libregrtest: Fix setup_tests() to keep module.__path__ type (_NamespacePath), don't convert to a list. Add _NamespacePath.__setitem__() method to importlib._bootstrap_external. --- Lib/importlib/_bootstrap_external.py | 3 + Lib/test/libregrtest/setup.py | 4 +- Python/importlib_external.h | 1871 +++++++++++++++++----------------- 3 files changed, 945 insertions(+), 933 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index ddb2456..5d63a1f 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -971,6 +971,9 @@ class _NamespacePath: def __iter__(self): return iter(self._recalculate()) + def __setitem__(self, index, path): + self._path[index] = path + def __len__(self): return len(self._recalculate()) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 6e05c7e..de52bb5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -41,8 +41,8 @@ def setup_tests(ns): # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): - module.__path__ = [os.path.abspath(path) - for path in module.__path__] + for index, path in enumerate(module.__path__): + module.__path__[index] = os.path.abspath(path) if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) diff --git a/Python/importlib_external.h b/Python/importlib_external.h index 3d7aff0..840e9c4 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -1637,7 +1637,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,0,0,115,20,0,0,0,12,6,6,2,12,4,12,4, 12,3,12,8,12,6,12,6,12,4,12,4,114,218,0,0, 0,99,0,0,0,0,0,0,0,0,0,0,0,0,2,0, - 0,0,64,0,0,0,115,130,0,0,0,101,0,0,90,1, + 0,0,64,0,0,0,115,142,0,0,0,101,0,0,90,1, 0,100,0,0,90,2,0,100,1,0,90,3,0,100,2,0, 100,3,0,132,0,0,90,4,0,100,4,0,100,5,0,132, 0,0,90,5,0,100,6,0,100,7,0,132,0,0,90,6, @@ -1645,939 +1645,948 @@ const unsigned char _Py_M__importlib_external[] = { 100,11,0,132,0,0,90,8,0,100,12,0,100,13,0,132, 0,0,90,9,0,100,14,0,100,15,0,132,0,0,90,10, 0,100,16,0,100,17,0,132,0,0,90,11,0,100,18,0, - 100,19,0,132,0,0,90,12,0,100,20,0,83,41,21,218, - 14,95,78,97,109,101,115,112,97,99,101,80,97,116,104,97, - 38,1,0,0,82,101,112,114,101,115,101,110,116,115,32,97, - 32,110,97,109,101,115,112,97,99,101,32,112,97,99,107,97, - 103,101,39,115,32,112,97,116,104,46,32,32,73,116,32,117, - 115,101,115,32,116,104,101,32,109,111,100,117,108,101,32,110, - 97,109,101,10,32,32,32,32,116,111,32,102,105,110,100,32, - 105,116,115,32,112,97,114,101,110,116,32,109,111,100,117,108, - 101,44,32,97,110,100,32,102,114,111,109,32,116,104,101,114, - 101,32,105,116,32,108,111,111,107,115,32,117,112,32,116,104, - 101,32,112,97,114,101,110,116,39,115,10,32,32,32,32,95, - 95,112,97,116,104,95,95,46,32,32,87,104,101,110,32,116, - 104,105,115,32,99,104,97,110,103,101,115,44,32,116,104,101, - 32,109,111,100,117,108,101,39,115,32,111,119,110,32,112,97, - 116,104,32,105,115,32,114,101,99,111,109,112,117,116,101,100, - 44,10,32,32,32,32,117,115,105,110,103,32,112,97,116,104, - 95,102,105,110,100,101,114,46,32,32,70,111,114,32,116,111, - 112,45,108,101,118,101,108,32,109,111,100,117,108,101,115,44, - 32,116,104,101,32,112,97,114,101,110,116,32,109,111,100,117, - 108,101,39,115,32,112,97,116,104,10,32,32,32,32,105,115, - 32,115,121,115,46,112,97,116,104,46,99,4,0,0,0,0, - 0,0,0,4,0,0,0,2,0,0,0,67,0,0,0,115, - 52,0,0,0,124,1,0,124,0,0,95,0,0,124,2,0, - 124,0,0,95,1,0,116,2,0,124,0,0,106,3,0,131, - 0,0,131,1,0,124,0,0,95,4,0,124,3,0,124,0, - 0,95,5,0,100,0,0,83,41,1,78,41,6,218,5,95, - 110,97,109,101,218,5,95,112,97,116,104,114,93,0,0,0, - 218,16,95,103,101,116,95,112,97,114,101,110,116,95,112,97, - 116,104,218,17,95,108,97,115,116,95,112,97,114,101,110,116, - 95,112,97,116,104,218,12,95,112,97,116,104,95,102,105,110, - 100,101,114,41,4,114,100,0,0,0,114,98,0,0,0,114, - 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, - 114,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,179,0,0,0,170,3,0,0,115,8,0,0,0,0,1, - 9,1,9,1,21,1,122,23,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, - 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, - 67,0,0,0,115,53,0,0,0,124,0,0,106,0,0,106, - 1,0,100,1,0,131,1,0,92,3,0,125,1,0,125,2, - 0,125,3,0,124,2,0,100,2,0,107,2,0,114,43,0, - 100,6,0,83,124,1,0,100,5,0,102,2,0,83,41,7, - 122,62,82,101,116,117,114,110,115,32,97,32,116,117,112,108, - 101,32,111,102,32,40,112,97,114,101,110,116,45,109,111,100, - 117,108,101,45,110,97,109,101,44,32,112,97,114,101,110,116, - 45,112,97,116,104,45,97,116,116,114,45,110,97,109,101,41, - 114,58,0,0,0,114,30,0,0,0,114,7,0,0,0,114, - 35,0,0,0,90,8,95,95,112,97,116,104,95,95,41,2, - 122,3,115,121,115,122,4,112,97,116,104,41,2,114,225,0, - 0,0,114,32,0,0,0,41,4,114,100,0,0,0,114,216, - 0,0,0,218,3,100,111,116,90,2,109,101,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,218,23,95,102,105, - 110,100,95,112,97,114,101,110,116,95,112,97,116,104,95,110, - 97,109,101,115,176,3,0,0,115,8,0,0,0,0,2,27, - 1,12,2,4,3,122,38,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,102,105,110,100,95,112,97,114,101, - 110,116,95,112,97,116,104,95,110,97,109,101,115,99,1,0, - 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, - 0,0,115,38,0,0,0,124,0,0,106,0,0,131,0,0, - 92,2,0,125,1,0,125,2,0,116,1,0,116,2,0,106, - 3,0,124,1,0,25,124,2,0,131,2,0,83,41,1,78, - 41,4,114,232,0,0,0,114,110,0,0,0,114,7,0,0, - 0,218,7,109,111,100,117,108,101,115,41,3,114,100,0,0, - 0,90,18,112,97,114,101,110,116,95,109,111,100,117,108,101, - 95,110,97,109,101,90,14,112,97,116,104,95,97,116,116,114, - 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,227,0,0,0,186,3,0,0,115,4,0, - 0,0,0,1,18,1,122,31,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,46,95,103,101,116,95,112,97,114,101, - 110,116,95,112,97,116,104,99,1,0,0,0,0,0,0,0, - 3,0,0,0,3,0,0,0,67,0,0,0,115,118,0,0, - 0,116,0,0,124,0,0,106,1,0,131,0,0,131,1,0, - 125,1,0,124,1,0,124,0,0,106,2,0,107,3,0,114, - 111,0,124,0,0,106,3,0,124,0,0,106,4,0,124,1, - 0,131,2,0,125,2,0,124,2,0,100,0,0,107,9,0, - 114,102,0,124,2,0,106,5,0,100,0,0,107,8,0,114, - 102,0,124,2,0,106,6,0,114,102,0,124,2,0,106,6, - 0,124,0,0,95,7,0,124,1,0,124,0,0,95,2,0, - 124,0,0,106,7,0,83,41,1,78,41,8,114,93,0,0, - 0,114,227,0,0,0,114,228,0,0,0,114,229,0,0,0, - 114,225,0,0,0,114,120,0,0,0,114,150,0,0,0,114, - 226,0,0,0,41,3,114,100,0,0,0,90,11,112,97,114, - 101,110,116,95,112,97,116,104,114,158,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,114, - 101,99,97,108,99,117,108,97,116,101,190,3,0,0,115,16, - 0,0,0,0,2,18,1,15,1,21,3,27,1,9,1,12, - 1,9,1,122,27,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,95,114,101,99,97,108,99,117,108,97,116,101, + 100,19,0,132,0,0,90,12,0,100,20,0,100,21,0,132, + 0,0,90,13,0,100,22,0,83,41,23,218,14,95,78,97, + 109,101,115,112,97,99,101,80,97,116,104,97,38,1,0,0, + 82,101,112,114,101,115,101,110,116,115,32,97,32,110,97,109, + 101,115,112,97,99,101,32,112,97,99,107,97,103,101,39,115, + 32,112,97,116,104,46,32,32,73,116,32,117,115,101,115,32, + 116,104,101,32,109,111,100,117,108,101,32,110,97,109,101,10, + 32,32,32,32,116,111,32,102,105,110,100,32,105,116,115,32, + 112,97,114,101,110,116,32,109,111,100,117,108,101,44,32,97, + 110,100,32,102,114,111,109,32,116,104,101,114,101,32,105,116, + 32,108,111,111,107,115,32,117,112,32,116,104,101,32,112,97, + 114,101,110,116,39,115,10,32,32,32,32,95,95,112,97,116, + 104,95,95,46,32,32,87,104,101,110,32,116,104,105,115,32, + 99,104,97,110,103,101,115,44,32,116,104,101,32,109,111,100, + 117,108,101,39,115,32,111,119,110,32,112,97,116,104,32,105, + 115,32,114,101,99,111,109,112,117,116,101,100,44,10,32,32, + 32,32,117,115,105,110,103,32,112,97,116,104,95,102,105,110, + 100,101,114,46,32,32,70,111,114,32,116,111,112,45,108,101, + 118,101,108,32,109,111,100,117,108,101,115,44,32,116,104,101, + 32,112,97,114,101,110,116,32,109,111,100,117,108,101,39,115, + 32,112,97,116,104,10,32,32,32,32,105,115,32,115,121,115, + 46,112,97,116,104,46,99,4,0,0,0,0,0,0,0,4, + 0,0,0,2,0,0,0,67,0,0,0,115,52,0,0,0, + 124,1,0,124,0,0,95,0,0,124,2,0,124,0,0,95, + 1,0,116,2,0,124,0,0,106,3,0,131,0,0,131,1, + 0,124,0,0,95,4,0,124,3,0,124,0,0,95,5,0, + 100,0,0,83,41,1,78,41,6,218,5,95,110,97,109,101, + 218,5,95,112,97,116,104,114,93,0,0,0,218,16,95,103, + 101,116,95,112,97,114,101,110,116,95,112,97,116,104,218,17, + 95,108,97,115,116,95,112,97,114,101,110,116,95,112,97,116, + 104,218,12,95,112,97,116,104,95,102,105,110,100,101,114,41, + 4,114,100,0,0,0,114,98,0,0,0,114,35,0,0,0, + 218,11,112,97,116,104,95,102,105,110,100,101,114,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,114,179,0,0, + 0,170,3,0,0,115,8,0,0,0,0,1,9,1,9,1, + 21,1,122,23,95,78,97,109,101,115,112,97,99,101,80,97, + 116,104,46,95,95,105,110,105,116,95,95,99,1,0,0,0, + 0,0,0,0,4,0,0,0,3,0,0,0,67,0,0,0, + 115,53,0,0,0,124,0,0,106,0,0,106,1,0,100,1, + 0,131,1,0,92,3,0,125,1,0,125,2,0,125,3,0, + 124,2,0,100,2,0,107,2,0,114,43,0,100,6,0,83, + 124,1,0,100,5,0,102,2,0,83,41,7,122,62,82,101, + 116,117,114,110,115,32,97,32,116,117,112,108,101,32,111,102, + 32,40,112,97,114,101,110,116,45,109,111,100,117,108,101,45, + 110,97,109,101,44,32,112,97,114,101,110,116,45,112,97,116, + 104,45,97,116,116,114,45,110,97,109,101,41,114,58,0,0, + 0,114,30,0,0,0,114,7,0,0,0,114,35,0,0,0, + 90,8,95,95,112,97,116,104,95,95,41,2,122,3,115,121, + 115,122,4,112,97,116,104,41,2,114,225,0,0,0,114,32, + 0,0,0,41,4,114,100,0,0,0,114,216,0,0,0,218, + 3,100,111,116,90,2,109,101,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,23,95,102,105,110,100,95,112, + 97,114,101,110,116,95,112,97,116,104,95,110,97,109,101,115, + 176,3,0,0,115,8,0,0,0,0,2,27,1,12,2,4, + 3,122,38,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,102,105,110,100,95,112,97,114,101,110,116,95,112, + 97,116,104,95,110,97,109,101,115,99,1,0,0,0,0,0, + 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,38, + 0,0,0,124,0,0,106,0,0,131,0,0,92,2,0,125, + 1,0,125,2,0,116,1,0,116,2,0,106,3,0,124,1, + 0,25,124,2,0,131,2,0,83,41,1,78,41,4,114,232, + 0,0,0,114,110,0,0,0,114,7,0,0,0,218,7,109, + 111,100,117,108,101,115,41,3,114,100,0,0,0,90,18,112, + 97,114,101,110,116,95,109,111,100,117,108,101,95,110,97,109, + 101,90,14,112,97,116,104,95,97,116,116,114,95,110,97,109, + 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,227,0,0,0,186,3,0,0,115,4,0,0,0,0,1, + 18,1,122,31,95,78,97,109,101,115,112,97,99,101,80,97, + 116,104,46,95,103,101,116,95,112,97,114,101,110,116,95,112, + 97,116,104,99,1,0,0,0,0,0,0,0,3,0,0,0, + 3,0,0,0,67,0,0,0,115,118,0,0,0,116,0,0, + 124,0,0,106,1,0,131,0,0,131,1,0,125,1,0,124, + 1,0,124,0,0,106,2,0,107,3,0,114,111,0,124,0, + 0,106,3,0,124,0,0,106,4,0,124,1,0,131,2,0, + 125,2,0,124,2,0,100,0,0,107,9,0,114,102,0,124, + 2,0,106,5,0,100,0,0,107,8,0,114,102,0,124,2, + 0,106,6,0,114,102,0,124,2,0,106,6,0,124,0,0, + 95,7,0,124,1,0,124,0,0,95,2,0,124,0,0,106, + 7,0,83,41,1,78,41,8,114,93,0,0,0,114,227,0, + 0,0,114,228,0,0,0,114,229,0,0,0,114,225,0,0, + 0,114,120,0,0,0,114,150,0,0,0,114,226,0,0,0, + 41,3,114,100,0,0,0,90,11,112,97,114,101,110,116,95, + 112,97,116,104,114,158,0,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,218,12,95,114,101,99,97,108, + 99,117,108,97,116,101,190,3,0,0,115,16,0,0,0,0, + 2,18,1,15,1,21,3,27,1,9,1,12,1,9,1,122, + 27,95,78,97,109,101,115,112,97,99,101,80,97,116,104,46, + 95,114,101,99,97,108,99,117,108,97,116,101,99,1,0,0, + 0,0,0,0,0,1,0,0,0,2,0,0,0,67,0,0, + 0,115,16,0,0,0,116,0,0,124,0,0,106,1,0,131, + 0,0,131,1,0,83,41,1,78,41,2,218,4,105,116,101, + 114,114,234,0,0,0,41,1,114,100,0,0,0,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,8,95,95, + 105,116,101,114,95,95,203,3,0,0,115,2,0,0,0,0, + 1,122,23,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,95,95,105,116,101,114,95,95,99,3,0,0,0,0, + 0,0,0,3,0,0,0,3,0,0,0,67,0,0,0,115, + 17,0,0,0,124,2,0,124,0,0,106,0,0,124,1,0, + 60,100,0,0,83,41,1,78,41,1,114,226,0,0,0,41, + 3,114,100,0,0,0,218,5,105,110,100,101,120,114,35,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,11,95,95,115,101,116,105,116,101,109,95,95,206,3, + 0,0,115,2,0,0,0,0,1,122,26,95,78,97,109,101, + 115,112,97,99,101,80,97,116,104,46,95,95,115,101,116,105, + 116,101,109,95,95,99,1,0,0,0,0,0,0,0,1,0, + 0,0,2,0,0,0,67,0,0,0,115,16,0,0,0,116, + 0,0,124,0,0,106,1,0,131,0,0,131,1,0,83,41, + 1,78,41,2,114,31,0,0,0,114,234,0,0,0,41,1, + 114,100,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,7,95,95,108,101,110,95,95,209,3,0, + 0,115,2,0,0,0,0,1,122,22,95,78,97,109,101,115, + 112,97,99,101,80,97,116,104,46,95,95,108,101,110,95,95, 99,1,0,0,0,0,0,0,0,1,0,0,0,2,0,0, - 0,67,0,0,0,115,16,0,0,0,116,0,0,124,0,0, - 106,1,0,131,0,0,131,1,0,83,41,1,78,41,2,218, - 4,105,116,101,114,114,234,0,0,0,41,1,114,100,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 218,8,95,95,105,116,101,114,95,95,203,3,0,0,115,2, - 0,0,0,0,1,122,23,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,95,105,116,101,114,95,95,99,1, - 0,0,0,0,0,0,0,1,0,0,0,2,0,0,0,67, - 0,0,0,115,16,0,0,0,116,0,0,124,0,0,106,1, - 0,131,0,0,131,1,0,83,41,1,78,41,2,114,31,0, - 0,0,114,234,0,0,0,41,1,114,100,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,218,7,95, - 95,108,101,110,95,95,206,3,0,0,115,2,0,0,0,0, - 1,122,22,95,78,97,109,101,115,112,97,99,101,80,97,116, - 104,46,95,95,108,101,110,95,95,99,1,0,0,0,0,0, - 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, - 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, - 1,0,83,41,2,78,122,20,95,78,97,109,101,115,112,97, - 99,101,80,97,116,104,40,123,33,114,125,41,41,2,114,47, - 0,0,0,114,226,0,0,0,41,1,114,100,0,0,0,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,95,114,101,112,114,95,95,209,3,0,0,115,2,0,0, - 0,0,1,122,23,95,78,97,109,101,115,112,97,99,101,80, - 97,116,104,46,95,95,114,101,112,114,95,95,99,2,0,0, - 0,0,0,0,0,2,0,0,0,2,0,0,0,67,0,0, - 0,115,16,0,0,0,124,1,0,124,0,0,106,0,0,131, - 0,0,107,6,0,83,41,1,78,41,1,114,234,0,0,0, - 41,2,114,100,0,0,0,218,4,105,116,101,109,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,12,95,95, - 99,111,110,116,97,105,110,115,95,95,212,3,0,0,115,2, - 0,0,0,0,1,122,27,95,78,97,109,101,115,112,97,99, - 101,80,97,116,104,46,95,95,99,111,110,116,97,105,110,115, - 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,2, - 0,0,0,67,0,0,0,115,20,0,0,0,124,0,0,106, - 0,0,106,1,0,124,1,0,131,1,0,1,100,0,0,83, - 41,1,78,41,2,114,226,0,0,0,114,157,0,0,0,41, - 2,114,100,0,0,0,114,239,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,157,0,0,0,215, - 3,0,0,115,2,0,0,0,0,1,122,21,95,78,97,109, - 101,115,112,97,99,101,80,97,116,104,46,97,112,112,101,110, - 100,78,41,13,114,105,0,0,0,114,104,0,0,0,114,106, - 0,0,0,114,107,0,0,0,114,179,0,0,0,114,232,0, - 0,0,114,227,0,0,0,114,234,0,0,0,114,236,0,0, - 0,114,237,0,0,0,114,238,0,0,0,114,240,0,0,0, - 114,157,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,114,224,0,0,0,163,3, - 0,0,115,20,0,0,0,12,5,6,2,12,6,12,10,12, - 4,12,13,12,3,12,3,12,3,12,3,114,224,0,0,0, - 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, - 0,64,0,0,0,115,118,0,0,0,101,0,0,90,1,0, - 100,0,0,90,2,0,100,1,0,100,2,0,132,0,0,90, - 3,0,101,4,0,100,3,0,100,4,0,132,0,0,131,1, - 0,90,5,0,100,5,0,100,6,0,132,0,0,90,6,0, - 100,7,0,100,8,0,132,0,0,90,7,0,100,9,0,100, - 10,0,132,0,0,90,8,0,100,11,0,100,12,0,132,0, - 0,90,9,0,100,13,0,100,14,0,132,0,0,90,10,0, - 100,15,0,100,16,0,132,0,0,90,11,0,100,17,0,83, - 41,18,218,16,95,78,97,109,101,115,112,97,99,101,76,111, - 97,100,101,114,99,4,0,0,0,0,0,0,0,4,0,0, - 0,4,0,0,0,67,0,0,0,115,25,0,0,0,116,0, - 0,124,1,0,124,2,0,124,3,0,131,3,0,124,0,0, - 95,1,0,100,0,0,83,41,1,78,41,2,114,224,0,0, - 0,114,226,0,0,0,41,4,114,100,0,0,0,114,98,0, - 0,0,114,35,0,0,0,114,230,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,179,0,0,0, - 221,3,0,0,115,2,0,0,0,0,1,122,25,95,78,97, - 109,101,115,112,97,99,101,76,111,97,100,101,114,46,95,95, - 105,110,105,116,95,95,99,2,0,0,0,0,0,0,0,2, - 0,0,0,2,0,0,0,67,0,0,0,115,16,0,0,0, - 100,1,0,106,0,0,124,1,0,106,1,0,131,1,0,83, - 41,2,122,115,82,101,116,117,114,110,32,114,101,112,114,32, - 102,111,114,32,116,104,101,32,109,111,100,117,108,101,46,10, - 10,32,32,32,32,32,32,32,32,84,104,101,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,84,104,101,32,105,109,112,111,114,116,32,109, - 97,99,104,105,110,101,114,121,32,100,111,101,115,32,116,104, - 101,32,106,111,98,32,105,116,115,101,108,102,46,10,10,32, - 32,32,32,32,32,32,32,122,25,60,109,111,100,117,108,101, - 32,123,33,114,125,32,40,110,97,109,101,115,112,97,99,101, - 41,62,41,2,114,47,0,0,0,114,105,0,0,0,41,2, - 114,164,0,0,0,114,184,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,11,109,111,100,117,108, - 101,95,114,101,112,114,224,3,0,0,115,2,0,0,0,0, - 7,122,28,95,78,97,109,101,115,112,97,99,101,76,111,97, - 100,101,114,46,109,111,100,117,108,101,95,114,101,112,114,99, - 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, - 67,0,0,0,115,4,0,0,0,100,1,0,83,41,2,78, - 84,114,4,0,0,0,41,2,114,100,0,0,0,114,119,0, + 0,67,0,0,0,115,16,0,0,0,100,1,0,106,0,0, + 124,0,0,106,1,0,131,1,0,83,41,2,78,122,20,95, + 78,97,109,101,115,112,97,99,101,80,97,116,104,40,123,33, + 114,125,41,41,2,114,47,0,0,0,114,226,0,0,0,41, + 1,114,100,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,218,8,95,95,114,101,112,114,95,95,212, + 3,0,0,115,2,0,0,0,0,1,122,23,95,78,97,109, + 101,115,112,97,99,101,80,97,116,104,46,95,95,114,101,112, + 114,95,95,99,2,0,0,0,0,0,0,0,2,0,0,0, + 2,0,0,0,67,0,0,0,115,16,0,0,0,124,1,0, + 124,0,0,106,0,0,131,0,0,107,6,0,83,41,1,78, + 41,1,114,234,0,0,0,41,2,114,100,0,0,0,218,4, + 105,116,101,109,114,4,0,0,0,114,4,0,0,0,114,5, + 0,0,0,218,12,95,95,99,111,110,116,97,105,110,115,95, + 95,215,3,0,0,115,2,0,0,0,0,1,122,27,95,78, + 97,109,101,115,112,97,99,101,80,97,116,104,46,95,95,99, + 111,110,116,97,105,110,115,95,95,99,2,0,0,0,0,0, + 0,0,2,0,0,0,2,0,0,0,67,0,0,0,115,20, + 0,0,0,124,0,0,106,0,0,106,1,0,124,1,0,131, + 1,0,1,100,0,0,83,41,1,78,41,2,114,226,0,0, + 0,114,157,0,0,0,41,2,114,100,0,0,0,114,241,0, 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,153,0,0,0,233,3,0,0,115,2,0,0,0,0, - 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, - 100,101,114,46,105,115,95,112,97,99,107,97,103,101,99,2, - 0,0,0,0,0,0,0,2,0,0,0,1,0,0,0,67, - 0,0,0,115,4,0,0,0,100,1,0,83,41,2,78,114, - 30,0,0,0,114,4,0,0,0,41,2,114,100,0,0,0, - 114,119,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,196,0,0,0,236,3,0,0,115,2,0, - 0,0,0,1,122,27,95,78,97,109,101,115,112,97,99,101, - 76,111,97,100,101,114,46,103,101,116,95,115,111,117,114,99, - 101,99,2,0,0,0,0,0,0,0,2,0,0,0,6,0, - 0,0,67,0,0,0,115,22,0,0,0,116,0,0,100,1, - 0,100,2,0,100,3,0,100,4,0,100,5,0,131,3,1, - 83,41,6,78,114,30,0,0,0,122,8,60,115,116,114,105, - 110,103,62,114,183,0,0,0,114,198,0,0,0,84,41,1, - 114,199,0,0,0,41,2,114,100,0,0,0,114,119,0,0, - 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,181,0,0,0,239,3,0,0,115,2,0,0,0,0,1, - 122,25,95,78,97,109,101,115,112,97,99,101,76,111,97,100, - 101,114,46,103,101,116,95,99,111,100,101,99,2,0,0,0, - 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, - 115,4,0,0,0,100,1,0,83,41,2,122,42,85,115,101, - 32,100,101,102,97,117,108,116,32,115,101,109,97,110,116,105, - 99,115,32,102,111,114,32,109,111,100,117,108,101,32,99,114, - 101,97,116,105,111,110,46,78,114,4,0,0,0,41,2,114, - 100,0,0,0,114,158,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,114,180,0,0,0,242,3,0, - 0,115,0,0,0,0,122,30,95,78,97,109,101,115,112,97, - 99,101,76,111,97,100,101,114,46,99,114,101,97,116,101,95, - 109,111,100,117,108,101,99,2,0,0,0,0,0,0,0,2, - 0,0,0,1,0,0,0,67,0,0,0,115,4,0,0,0, - 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,100, + 0,114,157,0,0,0,218,3,0,0,115,2,0,0,0,0, + 1,122,21,95,78,97,109,101,115,112,97,99,101,80,97,116, + 104,46,97,112,112,101,110,100,78,41,14,114,105,0,0,0, + 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, + 179,0,0,0,114,232,0,0,0,114,227,0,0,0,114,234, + 0,0,0,114,236,0,0,0,114,238,0,0,0,114,239,0, + 0,0,114,240,0,0,0,114,242,0,0,0,114,157,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,224,0,0,0,163,3,0,0,115,22, + 0,0,0,12,5,6,2,12,6,12,10,12,4,12,13,12, + 3,12,3,12,3,12,3,12,3,114,224,0,0,0,99,0, + 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, + 0,0,0,115,118,0,0,0,101,0,0,90,1,0,100,0, + 0,90,2,0,100,1,0,100,2,0,132,0,0,90,3,0, + 101,4,0,100,3,0,100,4,0,132,0,0,131,1,0,90, + 5,0,100,5,0,100,6,0,132,0,0,90,6,0,100,7, + 0,100,8,0,132,0,0,90,7,0,100,9,0,100,10,0, + 132,0,0,90,8,0,100,11,0,100,12,0,132,0,0,90, + 9,0,100,13,0,100,14,0,132,0,0,90,10,0,100,15, + 0,100,16,0,132,0,0,90,11,0,100,17,0,83,41,18, + 218,16,95,78,97,109,101,115,112,97,99,101,76,111,97,100, + 101,114,99,4,0,0,0,0,0,0,0,4,0,0,0,4, + 0,0,0,67,0,0,0,115,25,0,0,0,116,0,0,124, + 1,0,124,2,0,124,3,0,131,3,0,124,0,0,95,1, + 0,100,0,0,83,41,1,78,41,2,114,224,0,0,0,114, + 226,0,0,0,41,4,114,100,0,0,0,114,98,0,0,0, + 114,35,0,0,0,114,230,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,179,0,0,0,224,3, + 0,0,115,2,0,0,0,0,1,122,25,95,78,97,109,101, + 115,112,97,99,101,76,111,97,100,101,114,46,95,95,105,110, + 105,116,95,95,99,2,0,0,0,0,0,0,0,2,0,0, + 0,2,0,0,0,67,0,0,0,115,16,0,0,0,100,1, + 0,106,0,0,124,1,0,106,1,0,131,1,0,83,41,2, + 122,115,82,101,116,117,114,110,32,114,101,112,114,32,102,111, + 114,32,116,104,101,32,109,111,100,117,108,101,46,10,10,32, + 32,32,32,32,32,32,32,84,104,101,32,109,101,116,104,111, + 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, + 32,32,84,104,101,32,105,109,112,111,114,116,32,109,97,99, + 104,105,110,101,114,121,32,100,111,101,115,32,116,104,101,32, + 106,111,98,32,105,116,115,101,108,102,46,10,10,32,32,32, + 32,32,32,32,32,122,25,60,109,111,100,117,108,101,32,123, + 33,114,125,32,40,110,97,109,101,115,112,97,99,101,41,62, + 41,2,114,47,0,0,0,114,105,0,0,0,41,2,114,164, 0,0,0,114,184,0,0,0,114,4,0,0,0,114,4,0, - 0,0,114,5,0,0,0,114,185,0,0,0,245,3,0,0, - 115,2,0,0,0,0,1,122,28,95,78,97,109,101,115,112, - 97,99,101,76,111,97,100,101,114,46,101,120,101,99,95,109, - 111,100,117,108,101,99,2,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,35,0,0,0,116, - 0,0,106,1,0,100,1,0,124,0,0,106,2,0,131,2, - 0,1,116,0,0,106,3,0,124,0,0,124,1,0,131,2, - 0,83,41,2,122,98,76,111,97,100,32,97,32,110,97,109, - 101,115,112,97,99,101,32,109,111,100,117,108,101,46,10,10, - 32,32,32,32,32,32,32,32,84,104,105,115,32,109,101,116, - 104,111,100,32,105,115,32,100,101,112,114,101,99,97,116,101, - 100,46,32,32,85,115,101,32,101,120,101,99,95,109,111,100, - 117,108,101,40,41,32,105,110,115,116,101,97,100,46,10,10, - 32,32,32,32,32,32,32,32,122,38,110,97,109,101,115,112, - 97,99,101,32,109,111,100,117,108,101,32,108,111,97,100,101, - 100,32,119,105,116,104,32,112,97,116,104,32,123,33,114,125, - 41,4,114,114,0,0,0,114,129,0,0,0,114,226,0,0, - 0,114,186,0,0,0,41,2,114,100,0,0,0,114,119,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,114,187,0,0,0,248,3,0,0,115,6,0,0,0,0, - 7,9,1,10,1,122,28,95,78,97,109,101,115,112,97,99, - 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, - 117,108,101,78,41,12,114,105,0,0,0,114,104,0,0,0, - 114,106,0,0,0,114,179,0,0,0,114,177,0,0,0,114, - 242,0,0,0,114,153,0,0,0,114,196,0,0,0,114,181, - 0,0,0,114,180,0,0,0,114,185,0,0,0,114,187,0, - 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,241,0,0,0,220,3,0,0,115, - 16,0,0,0,12,1,12,3,18,9,12,3,12,3,12,3, - 12,3,12,3,114,241,0,0,0,99,0,0,0,0,0,0, - 0,0,0,0,0,0,5,0,0,0,64,0,0,0,115,160, - 0,0,0,101,0,0,90,1,0,100,0,0,90,2,0,100, - 1,0,90,3,0,101,4,0,100,2,0,100,3,0,132,0, - 0,131,1,0,90,5,0,101,4,0,100,4,0,100,5,0, - 132,0,0,131,1,0,90,6,0,101,4,0,100,6,0,100, - 7,0,132,0,0,131,1,0,90,7,0,101,4,0,100,8, - 0,100,9,0,132,0,0,131,1,0,90,8,0,101,4,0, - 100,10,0,100,11,0,100,12,0,132,1,0,131,1,0,90, - 9,0,101,4,0,100,10,0,100,10,0,100,13,0,100,14, - 0,132,2,0,131,1,0,90,10,0,101,4,0,100,10,0, - 100,15,0,100,16,0,132,1,0,131,1,0,90,11,0,100, - 10,0,83,41,17,218,10,80,97,116,104,70,105,110,100,101, - 114,122,62,77,101,116,97,32,112,97,116,104,32,102,105,110, - 100,101,114,32,102,111,114,32,115,121,115,46,112,97,116,104, - 32,97,110,100,32,112,97,99,107,97,103,101,32,95,95,112, - 97,116,104,95,95,32,97,116,116,114,105,98,117,116,101,115, - 46,99,1,0,0,0,0,0,0,0,2,0,0,0,4,0, - 0,0,67,0,0,0,115,55,0,0,0,120,48,0,116,0, - 0,106,1,0,106,2,0,131,0,0,68,93,31,0,125,1, - 0,116,3,0,124,1,0,100,1,0,131,2,0,114,16,0, - 124,1,0,106,4,0,131,0,0,1,113,16,0,87,100,2, - 0,83,41,3,122,125,67,97,108,108,32,116,104,101,32,105, - 110,118,97,108,105,100,97,116,101,95,99,97,99,104,101,115, - 40,41,32,109,101,116,104,111,100,32,111,110,32,97,108,108, - 32,112,97,116,104,32,101,110,116,114,121,32,102,105,110,100, - 101,114,115,10,32,32,32,32,32,32,32,32,115,116,111,114, - 101,100,32,105,110,32,115,121,115,46,112,97,116,104,95,105, - 109,112,111,114,116,101,114,95,99,97,99,104,101,115,32,40, - 119,104,101,114,101,32,105,109,112,108,101,109,101,110,116,101, - 100,41,46,218,17,105,110,118,97,108,105,100,97,116,101,95, - 99,97,99,104,101,115,78,41,5,114,7,0,0,0,218,19, - 112,97,116,104,95,105,109,112,111,114,116,101,114,95,99,97, - 99,104,101,218,6,118,97,108,117,101,115,114,108,0,0,0, - 114,244,0,0,0,41,2,114,164,0,0,0,218,6,102,105, - 110,100,101,114,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,114,244,0,0,0,10,4,0,0,115,6,0,0, - 0,0,4,22,1,15,1,122,28,80,97,116,104,70,105,110, - 100,101,114,46,105,110,118,97,108,105,100,97,116,101,95,99, - 97,99,104,101,115,99,2,0,0,0,0,0,0,0,3,0, - 0,0,12,0,0,0,67,0,0,0,115,107,0,0,0,116, - 0,0,106,1,0,100,1,0,107,9,0,114,41,0,116,0, - 0,106,1,0,12,114,41,0,116,2,0,106,3,0,100,2, - 0,116,4,0,131,2,0,1,120,59,0,116,0,0,106,1, - 0,68,93,44,0,125,2,0,121,14,0,124,2,0,124,1, - 0,131,1,0,83,87,113,51,0,4,116,5,0,107,10,0, - 114,94,0,1,1,1,119,51,0,89,113,51,0,88,113,51, - 0,87,100,1,0,83,100,1,0,83,41,3,122,113,83,101, - 97,114,99,104,32,115,101,113,117,101,110,99,101,32,111,102, - 32,104,111,111,107,115,32,102,111,114,32,97,32,102,105,110, - 100,101,114,32,102,111,114,32,39,112,97,116,104,39,46,10, - 10,32,32,32,32,32,32,32,32,73,102,32,39,104,111,111, - 107,115,39,32,105,115,32,102,97,108,115,101,32,116,104,101, - 110,32,117,115,101,32,115,121,115,46,112,97,116,104,95,104, - 111,111,107,115,46,10,10,32,32,32,32,32,32,32,32,78, - 122,23,115,121,115,46,112,97,116,104,95,104,111,111,107,115, - 32,105,115,32,101,109,112,116,121,41,6,114,7,0,0,0, - 218,10,112,97,116,104,95,104,111,111,107,115,114,60,0,0, - 0,114,61,0,0,0,114,118,0,0,0,114,99,0,0,0, - 41,3,114,164,0,0,0,114,35,0,0,0,90,4,104,111, - 111,107,114,4,0,0,0,114,4,0,0,0,114,5,0,0, - 0,218,11,95,112,97,116,104,95,104,111,111,107,115,18,4, - 0,0,115,16,0,0,0,0,7,25,1,16,1,16,1,3, - 1,14,1,13,1,12,2,122,22,80,97,116,104,70,105,110, - 100,101,114,46,95,112,97,116,104,95,104,111,111,107,115,99, - 2,0,0,0,0,0,0,0,3,0,0,0,19,0,0,0, - 67,0,0,0,115,123,0,0,0,124,1,0,100,1,0,107, - 2,0,114,53,0,121,16,0,116,0,0,106,1,0,131,0, - 0,125,1,0,87,110,22,0,4,116,2,0,107,10,0,114, - 52,0,1,1,1,100,2,0,83,89,110,1,0,88,121,17, - 0,116,3,0,106,4,0,124,1,0,25,125,2,0,87,110, - 46,0,4,116,5,0,107,10,0,114,118,0,1,1,1,124, - 0,0,106,6,0,124,1,0,131,1,0,125,2,0,124,2, - 0,116,3,0,106,4,0,124,1,0,60,89,110,1,0,88, - 124,2,0,83,41,3,122,210,71,101,116,32,116,104,101,32, - 102,105,110,100,101,114,32,102,111,114,32,116,104,101,32,112, - 97,116,104,32,101,110,116,114,121,32,102,114,111,109,32,115, - 121,115,46,112,97,116,104,95,105,109,112,111,114,116,101,114, - 95,99,97,99,104,101,46,10,10,32,32,32,32,32,32,32, - 32,73,102,32,116,104,101,32,112,97,116,104,32,101,110,116, - 114,121,32,105,115,32,110,111,116,32,105,110,32,116,104,101, - 32,99,97,99,104,101,44,32,102,105,110,100,32,116,104,101, - 32,97,112,112,114,111,112,114,105,97,116,101,32,102,105,110, - 100,101,114,10,32,32,32,32,32,32,32,32,97,110,100,32, - 99,97,99,104,101,32,105,116,46,32,73,102,32,110,111,32, - 102,105,110,100,101,114,32,105,115,32,97,118,97,105,108,97, - 98,108,101,44,32,115,116,111,114,101,32,78,111,110,101,46, - 10,10,32,32,32,32,32,32,32,32,114,30,0,0,0,78, - 41,7,114,3,0,0,0,114,45,0,0,0,218,17,70,105, - 108,101,78,111,116,70,111,117,110,100,69,114,114,111,114,114, - 7,0,0,0,114,245,0,0,0,114,131,0,0,0,114,249, - 0,0,0,41,3,114,164,0,0,0,114,35,0,0,0,114, - 247,0,0,0,114,4,0,0,0,114,4,0,0,0,114,5, - 0,0,0,218,20,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,35,4,0,0,115,22,0, - 0,0,0,8,12,1,3,1,16,1,13,3,9,1,3,1, - 17,1,13,1,15,1,18,1,122,31,80,97,116,104,70,105, - 110,100,101,114,46,95,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,99,3,0,0,0,0,0, - 0,0,6,0,0,0,3,0,0,0,67,0,0,0,115,119, - 0,0,0,116,0,0,124,2,0,100,1,0,131,2,0,114, - 39,0,124,2,0,106,1,0,124,1,0,131,1,0,92,2, - 0,125,3,0,125,4,0,110,21,0,124,2,0,106,2,0, - 124,1,0,131,1,0,125,3,0,103,0,0,125,4,0,124, - 3,0,100,0,0,107,9,0,114,88,0,116,3,0,106,4, - 0,124,1,0,124,3,0,131,2,0,83,116,3,0,106,5, - 0,124,1,0,100,0,0,131,2,0,125,5,0,124,4,0, - 124,5,0,95,6,0,124,5,0,83,41,2,78,114,117,0, - 0,0,41,7,114,108,0,0,0,114,117,0,0,0,114,176, - 0,0,0,114,114,0,0,0,114,173,0,0,0,114,154,0, - 0,0,114,150,0,0,0,41,6,114,164,0,0,0,114,119, - 0,0,0,114,247,0,0,0,114,120,0,0,0,114,121,0, + 0,0,114,5,0,0,0,218,11,109,111,100,117,108,101,95, + 114,101,112,114,227,3,0,0,115,2,0,0,0,0,7,122, + 28,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, + 114,46,109,111,100,117,108,101,95,114,101,112,114,99,2,0, + 0,0,0,0,0,0,2,0,0,0,1,0,0,0,67,0, + 0,0,115,4,0,0,0,100,1,0,83,41,2,78,84,114, + 4,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 153,0,0,0,236,3,0,0,115,2,0,0,0,0,1,122, + 27,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, + 114,46,105,115,95,112,97,99,107,97,103,101,99,2,0,0, + 0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0, + 0,115,4,0,0,0,100,1,0,83,41,2,78,114,30,0, + 0,0,114,4,0,0,0,41,2,114,100,0,0,0,114,119, + 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, + 0,0,114,196,0,0,0,239,3,0,0,115,2,0,0,0, + 0,1,122,27,95,78,97,109,101,115,112,97,99,101,76,111, + 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, + 2,0,0,0,0,0,0,0,2,0,0,0,6,0,0,0, + 67,0,0,0,115,22,0,0,0,116,0,0,100,1,0,100, + 2,0,100,3,0,100,4,0,100,5,0,131,3,1,83,41, + 6,78,114,30,0,0,0,122,8,60,115,116,114,105,110,103, + 62,114,183,0,0,0,114,198,0,0,0,84,41,1,114,199, + 0,0,0,41,2,114,100,0,0,0,114,119,0,0,0,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,181, + 0,0,0,242,3,0,0,115,2,0,0,0,0,1,122,25, + 95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,114, + 46,103,101,116,95,99,111,100,101,99,2,0,0,0,0,0, + 0,0,2,0,0,0,1,0,0,0,67,0,0,0,115,4, + 0,0,0,100,1,0,83,41,2,122,42,85,115,101,32,100, + 101,102,97,117,108,116,32,115,101,109,97,110,116,105,99,115, + 32,102,111,114,32,109,111,100,117,108,101,32,99,114,101,97, + 116,105,111,110,46,78,114,4,0,0,0,41,2,114,100,0, 0,0,114,158,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,218,16,95,108,101,103,97,99,121,95, - 103,101,116,95,115,112,101,99,57,4,0,0,115,18,0,0, - 0,0,4,15,1,24,2,15,1,6,1,12,1,16,1,18, - 1,9,1,122,27,80,97,116,104,70,105,110,100,101,114,46, - 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, - 78,99,4,0,0,0,0,0,0,0,9,0,0,0,5,0, - 0,0,67,0,0,0,115,243,0,0,0,103,0,0,125,4, - 0,120,230,0,124,2,0,68,93,191,0,125,5,0,116,0, - 0,124,5,0,116,1,0,116,2,0,102,2,0,131,2,0, - 115,43,0,113,13,0,124,0,0,106,3,0,124,5,0,131, - 1,0,125,6,0,124,6,0,100,1,0,107,9,0,114,13, - 0,116,4,0,124,6,0,100,2,0,131,2,0,114,106,0, - 124,6,0,106,5,0,124,1,0,124,3,0,131,2,0,125, - 7,0,110,18,0,124,0,0,106,6,0,124,1,0,124,6, - 0,131,2,0,125,7,0,124,7,0,100,1,0,107,8,0, - 114,139,0,113,13,0,124,7,0,106,7,0,100,1,0,107, - 9,0,114,158,0,124,7,0,83,124,7,0,106,8,0,125, - 8,0,124,8,0,100,1,0,107,8,0,114,191,0,116,9, - 0,100,3,0,131,1,0,130,1,0,124,4,0,106,10,0, - 124,8,0,131,1,0,1,113,13,0,87,116,11,0,106,12, - 0,124,1,0,100,1,0,131,2,0,125,7,0,124,4,0, - 124,7,0,95,8,0,124,7,0,83,100,1,0,83,41,4, - 122,63,70,105,110,100,32,116,104,101,32,108,111,97,100,101, - 114,32,111,114,32,110,97,109,101,115,112,97,99,101,95,112, - 97,116,104,32,102,111,114,32,116,104,105,115,32,109,111,100, - 117,108,101,47,112,97,99,107,97,103,101,32,110,97,109,101, - 46,78,114,175,0,0,0,122,19,115,112,101,99,32,109,105, - 115,115,105,110,103,32,108,111,97,100,101,114,41,13,114,137, - 0,0,0,114,69,0,0,0,218,5,98,121,116,101,115,114, - 251,0,0,0,114,108,0,0,0,114,175,0,0,0,114,252, - 0,0,0,114,120,0,0,0,114,150,0,0,0,114,99,0, - 0,0,114,143,0,0,0,114,114,0,0,0,114,154,0,0, - 0,41,9,114,164,0,0,0,114,119,0,0,0,114,35,0, - 0,0,114,174,0,0,0,218,14,110,97,109,101,115,112,97, - 99,101,95,112,97,116,104,90,5,101,110,116,114,121,114,247, - 0,0,0,114,158,0,0,0,114,121,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,218,9,95,103, - 101,116,95,115,112,101,99,72,4,0,0,115,40,0,0,0, - 0,5,6,1,13,1,21,1,3,1,15,1,12,1,15,1, - 21,2,18,1,12,1,3,1,15,1,4,1,9,1,12,1, - 12,5,17,2,18,1,9,1,122,20,80,97,116,104,70,105, - 110,100,101,114,46,95,103,101,116,95,115,112,101,99,99,4, - 0,0,0,0,0,0,0,6,0,0,0,4,0,0,0,67, - 0,0,0,115,140,0,0,0,124,2,0,100,1,0,107,8, - 0,114,21,0,116,0,0,106,1,0,125,2,0,124,0,0, - 106,2,0,124,1,0,124,2,0,124,3,0,131,3,0,125, - 4,0,124,4,0,100,1,0,107,8,0,114,58,0,100,1, - 0,83,124,4,0,106,3,0,100,1,0,107,8,0,114,132, - 0,124,4,0,106,4,0,125,5,0,124,5,0,114,125,0, - 100,2,0,124,4,0,95,5,0,116,6,0,124,1,0,124, - 5,0,124,0,0,106,2,0,131,3,0,124,4,0,95,4, - 0,124,4,0,83,100,1,0,83,110,4,0,124,4,0,83, - 100,1,0,83,41,3,122,98,102,105,110,100,32,116,104,101, - 32,109,111,100,117,108,101,32,111,110,32,115,121,115,46,112, - 97,116,104,32,111,114,32,39,112,97,116,104,39,32,98,97, - 115,101,100,32,111,110,32,115,121,115,46,112,97,116,104,95, - 104,111,111,107,115,32,97,110,100,10,32,32,32,32,32,32, - 32,32,115,121,115,46,112,97,116,104,95,105,109,112,111,114, - 116,101,114,95,99,97,99,104,101,46,78,90,9,110,97,109, - 101,115,112,97,99,101,41,7,114,7,0,0,0,114,35,0, - 0,0,114,255,0,0,0,114,120,0,0,0,114,150,0,0, - 0,114,152,0,0,0,114,224,0,0,0,41,6,114,164,0, - 0,0,114,119,0,0,0,114,35,0,0,0,114,174,0,0, - 0,114,158,0,0,0,114,254,0,0,0,114,4,0,0,0, - 114,4,0,0,0,114,5,0,0,0,114,175,0,0,0,104, - 4,0,0,115,26,0,0,0,0,4,12,1,9,1,21,1, - 12,1,4,1,15,1,9,1,6,3,9,1,24,1,4,2, - 7,2,122,20,80,97,116,104,70,105,110,100,101,114,46,102, - 105,110,100,95,115,112,101,99,99,3,0,0,0,0,0,0, - 0,4,0,0,0,3,0,0,0,67,0,0,0,115,41,0, - 0,0,124,0,0,106,0,0,124,1,0,124,2,0,131,2, - 0,125,3,0,124,3,0,100,1,0,107,8,0,114,34,0, - 100,1,0,83,124,3,0,106,1,0,83,41,2,122,170,102, - 105,110,100,32,116,104,101,32,109,111,100,117,108,101,32,111, - 110,32,115,121,115,46,112,97,116,104,32,111,114,32,39,112, - 97,116,104,39,32,98,97,115,101,100,32,111,110,32,115,121, - 115,46,112,97,116,104,95,104,111,111,107,115,32,97,110,100, - 10,32,32,32,32,32,32,32,32,115,121,115,46,112,97,116, - 104,95,105,109,112,111,114,116,101,114,95,99,97,99,104,101, - 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, - 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, - 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, - 10,32,32,32,32,32,32,32,32,78,41,2,114,175,0,0, - 0,114,120,0,0,0,41,4,114,164,0,0,0,114,119,0, - 0,0,114,35,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,176,0,0,0, - 126,4,0,0,115,8,0,0,0,0,8,18,1,12,1,4, - 1,122,22,80,97,116,104,70,105,110,100,101,114,46,102,105, - 110,100,95,109,111,100,117,108,101,41,12,114,105,0,0,0, - 114,104,0,0,0,114,106,0,0,0,114,107,0,0,0,114, - 177,0,0,0,114,244,0,0,0,114,249,0,0,0,114,251, - 0,0,0,114,252,0,0,0,114,255,0,0,0,114,175,0, - 0,0,114,176,0,0,0,114,4,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,243,0,0,0, - 6,4,0,0,115,22,0,0,0,12,2,6,2,18,8,18, - 17,18,22,18,15,3,1,18,31,3,1,21,21,3,1,114, - 243,0,0,0,99,0,0,0,0,0,0,0,0,0,0,0, - 0,3,0,0,0,64,0,0,0,115,133,0,0,0,101,0, - 0,90,1,0,100,0,0,90,2,0,100,1,0,90,3,0, - 100,2,0,100,3,0,132,0,0,90,4,0,100,4,0,100, - 5,0,132,0,0,90,5,0,101,6,0,90,7,0,100,6, - 0,100,7,0,132,0,0,90,8,0,100,8,0,100,9,0, - 132,0,0,90,9,0,100,10,0,100,11,0,100,12,0,132, - 1,0,90,10,0,100,13,0,100,14,0,132,0,0,90,11, - 0,101,12,0,100,15,0,100,16,0,132,0,0,131,1,0, - 90,13,0,100,17,0,100,18,0,132,0,0,90,14,0,100, - 10,0,83,41,19,218,10,70,105,108,101,70,105,110,100,101, - 114,122,172,70,105,108,101,45,98,97,115,101,100,32,102,105, - 110,100,101,114,46,10,10,32,32,32,32,73,110,116,101,114, - 97,99,116,105,111,110,115,32,119,105,116,104,32,116,104,101, - 32,102,105,108,101,32,115,121,115,116,101,109,32,97,114,101, - 32,99,97,99,104,101,100,32,102,111,114,32,112,101,114,102, - 111,114,109,97,110,99,101,44,32,98,101,105,110,103,10,32, - 32,32,32,114,101,102,114,101,115,104,101,100,32,119,104,101, - 110,32,116,104,101,32,100,105,114,101,99,116,111,114,121,32, - 116,104,101,32,102,105,110,100,101,114,32,105,115,32,104,97, - 110,100,108,105,110,103,32,104,97,115,32,98,101,101,110,32, - 109,111,100,105,102,105,101,100,46,10,10,32,32,32,32,99, - 2,0,0,0,0,0,0,0,5,0,0,0,5,0,0,0, - 7,0,0,0,115,122,0,0,0,103,0,0,125,3,0,120, - 52,0,124,2,0,68,93,44,0,92,2,0,137,0,0,125, - 4,0,124,3,0,106,0,0,135,0,0,102,1,0,100,1, - 0,100,2,0,134,0,0,124,4,0,68,131,1,0,131,1, - 0,1,113,13,0,87,124,3,0,124,0,0,95,1,0,124, - 1,0,112,79,0,100,3,0,124,0,0,95,2,0,100,6, - 0,124,0,0,95,3,0,116,4,0,131,0,0,124,0,0, - 95,5,0,116,4,0,131,0,0,124,0,0,95,6,0,100, - 5,0,83,41,7,122,154,73,110,105,116,105,97,108,105,122, - 101,32,119,105,116,104,32,116,104,101,32,112,97,116,104,32, - 116,111,32,115,101,97,114,99,104,32,111,110,32,97,110,100, - 32,97,32,118,97,114,105,97,98,108,101,32,110,117,109,98, - 101,114,32,111,102,10,32,32,32,32,32,32,32,32,50,45, - 116,117,112,108,101,115,32,99,111,110,116,97,105,110,105,110, - 103,32,116,104,101,32,108,111,97,100,101,114,32,97,110,100, - 32,116,104,101,32,102,105,108,101,32,115,117,102,102,105,120, - 101,115,32,116,104,101,32,108,111,97,100,101,114,10,32,32, - 32,32,32,32,32,32,114,101,99,111,103,110,105,122,101,115, - 46,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, - 0,0,51,0,0,0,115,27,0,0,0,124,0,0,93,17, - 0,125,1,0,124,1,0,136,0,0,102,2,0,86,1,113, - 3,0,100,0,0,83,41,1,78,114,4,0,0,0,41,2, - 114,22,0,0,0,114,219,0,0,0,41,1,114,120,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 155,4,0,0,115,2,0,0,0,6,0,122,38,70,105,108, - 101,70,105,110,100,101,114,46,95,95,105,110,105,116,95,95, - 46,60,108,111,99,97,108,115,62,46,60,103,101,110,101,120, - 112,114,62,114,58,0,0,0,114,29,0,0,0,78,114,87, - 0,0,0,41,7,114,143,0,0,0,218,8,95,108,111,97, - 100,101,114,115,114,35,0,0,0,218,11,95,112,97,116,104, - 95,109,116,105,109,101,218,3,115,101,116,218,11,95,112,97, - 116,104,95,99,97,99,104,101,218,19,95,114,101,108,97,120, - 101,100,95,112,97,116,104,95,99,97,99,104,101,41,5,114, - 100,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, - 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, - 114,115,114,160,0,0,0,114,4,0,0,0,41,1,114,120, - 0,0,0,114,5,0,0,0,114,179,0,0,0,149,4,0, - 0,115,16,0,0,0,0,4,6,1,19,1,36,1,9,2, - 15,1,9,1,12,1,122,19,70,105,108,101,70,105,110,100, - 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, - 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, - 115,13,0,0,0,100,3,0,124,0,0,95,0,0,100,2, - 0,83,41,4,122,31,73,110,118,97,108,105,100,97,116,101, - 32,116,104,101,32,100,105,114,101,99,116,111,114,121,32,109, - 116,105,109,101,46,114,29,0,0,0,78,114,87,0,0,0, - 41,1,114,2,1,0,0,41,1,114,100,0,0,0,114,4, - 0,0,0,114,4,0,0,0,114,5,0,0,0,114,244,0, - 0,0,163,4,0,0,115,2,0,0,0,0,2,122,28,70, - 105,108,101,70,105,110,100,101,114,46,105,110,118,97,108,105, - 100,97,116,101,95,99,97,99,104,101,115,99,2,0,0,0, - 0,0,0,0,3,0,0,0,2,0,0,0,67,0,0,0, - 115,59,0,0,0,124,0,0,106,0,0,124,1,0,131,1, - 0,125,2,0,124,2,0,100,1,0,107,8,0,114,37,0, - 100,1,0,103,0,0,102,2,0,83,124,2,0,106,1,0, - 124,2,0,106,2,0,112,55,0,103,0,0,102,2,0,83, - 41,2,122,197,84,114,121,32,116,111,32,102,105,110,100,32, - 97,32,108,111,97,100,101,114,32,102,111,114,32,116,104,101, - 32,115,112,101,99,105,102,105,101,100,32,109,111,100,117,108, - 101,44,32,111,114,32,116,104,101,32,110,97,109,101,115,112, - 97,99,101,10,32,32,32,32,32,32,32,32,112,97,99,107, - 97,103,101,32,112,111,114,116,105,111,110,115,46,32,82,101, - 116,117,114,110,115,32,40,108,111,97,100,101,114,44,32,108, - 105,115,116,45,111,102,45,112,111,114,116,105,111,110,115,41, - 46,10,10,32,32,32,32,32,32,32,32,84,104,105,115,32, - 109,101,116,104,111,100,32,105,115,32,100,101,112,114,101,99, - 97,116,101,100,46,32,32,85,115,101,32,102,105,110,100,95, - 115,112,101,99,40,41,32,105,110,115,116,101,97,100,46,10, - 10,32,32,32,32,32,32,32,32,78,41,3,114,175,0,0, - 0,114,120,0,0,0,114,150,0,0,0,41,3,114,100,0, - 0,0,114,119,0,0,0,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,117,0,0,0, - 169,4,0,0,115,8,0,0,0,0,7,15,1,12,1,10, - 1,122,22,70,105,108,101,70,105,110,100,101,114,46,102,105, - 110,100,95,108,111,97,100,101,114,99,6,0,0,0,0,0, - 0,0,7,0,0,0,7,0,0,0,67,0,0,0,115,40, - 0,0,0,124,1,0,124,2,0,124,3,0,131,2,0,125, - 6,0,116,0,0,124,2,0,124,3,0,100,1,0,124,6, - 0,100,2,0,124,4,0,131,2,2,83,41,3,78,114,120, - 0,0,0,114,150,0,0,0,41,1,114,161,0,0,0,41, - 7,114,100,0,0,0,114,159,0,0,0,114,119,0,0,0, - 114,35,0,0,0,90,4,115,109,115,108,114,174,0,0,0, - 114,120,0,0,0,114,4,0,0,0,114,4,0,0,0,114, - 5,0,0,0,114,255,0,0,0,181,4,0,0,115,6,0, - 0,0,0,1,15,1,18,1,122,20,70,105,108,101,70,105, - 110,100,101,114,46,95,103,101,116,95,115,112,101,99,78,99, - 3,0,0,0,0,0,0,0,14,0,0,0,15,0,0,0, - 67,0,0,0,115,228,1,0,0,100,1,0,125,3,0,124, - 1,0,106,0,0,100,2,0,131,1,0,100,3,0,25,125, - 4,0,121,34,0,116,1,0,124,0,0,106,2,0,112,49, - 0,116,3,0,106,4,0,131,0,0,131,1,0,106,5,0, - 125,5,0,87,110,24,0,4,116,6,0,107,10,0,114,85, - 0,1,1,1,100,10,0,125,5,0,89,110,1,0,88,124, - 5,0,124,0,0,106,7,0,107,3,0,114,120,0,124,0, - 0,106,8,0,131,0,0,1,124,5,0,124,0,0,95,7, - 0,116,9,0,131,0,0,114,153,0,124,0,0,106,10,0, - 125,6,0,124,4,0,106,11,0,131,0,0,125,7,0,110, - 15,0,124,0,0,106,12,0,125,6,0,124,4,0,125,7, - 0,124,7,0,124,6,0,107,6,0,114,45,1,116,13,0, - 124,0,0,106,2,0,124,4,0,131,2,0,125,8,0,120, - 100,0,124,0,0,106,14,0,68,93,77,0,92,2,0,125, - 9,0,125,10,0,100,5,0,124,9,0,23,125,11,0,116, - 13,0,124,8,0,124,11,0,131,2,0,125,12,0,116,15, - 0,124,12,0,131,1,0,114,208,0,124,0,0,106,16,0, - 124,10,0,124,1,0,124,12,0,124,8,0,103,1,0,124, - 2,0,131,5,0,83,113,208,0,87,116,17,0,124,8,0, - 131,1,0,125,3,0,120,120,0,124,0,0,106,14,0,68, - 93,109,0,92,2,0,125,9,0,125,10,0,116,13,0,124, - 0,0,106,2,0,124,4,0,124,9,0,23,131,2,0,125, - 12,0,116,18,0,106,19,0,100,6,0,124,12,0,100,7, - 0,100,3,0,131,2,1,1,124,7,0,124,9,0,23,124, - 6,0,107,6,0,114,55,1,116,15,0,124,12,0,131,1, - 0,114,55,1,124,0,0,106,16,0,124,10,0,124,1,0, - 124,12,0,100,8,0,124,2,0,131,5,0,83,113,55,1, - 87,124,3,0,114,224,1,116,18,0,106,19,0,100,9,0, - 124,8,0,131,2,0,1,116,18,0,106,20,0,124,1,0, - 100,8,0,131,2,0,125,13,0,124,8,0,103,1,0,124, - 13,0,95,21,0,124,13,0,83,100,8,0,83,41,11,122, - 125,84,114,121,32,116,111,32,102,105,110,100,32,97,32,108, - 111,97,100,101,114,32,102,111,114,32,116,104,101,32,115,112, - 101,99,105,102,105,101,100,32,109,111,100,117,108,101,44,32, - 111,114,32,116,104,101,32,110,97,109,101,115,112,97,99,101, - 10,32,32,32,32,32,32,32,32,112,97,99,107,97,103,101, - 32,112,111,114,116,105,111,110,115,46,32,82,101,116,117,114, - 110,115,32,40,108,111,97,100,101,114,44,32,108,105,115,116, - 45,111,102,45,112,111,114,116,105,111,110,115,41,46,70,114, - 58,0,0,0,114,56,0,0,0,114,29,0,0,0,114,179, - 0,0,0,122,9,116,114,121,105,110,103,32,123,125,90,9, - 118,101,114,98,111,115,105,116,121,78,122,25,112,111,115,115, - 105,98,108,101,32,110,97,109,101,115,112,97,99,101,32,102, - 111,114,32,123,125,114,87,0,0,0,41,22,114,32,0,0, - 0,114,39,0,0,0,114,35,0,0,0,114,3,0,0,0, - 114,45,0,0,0,114,213,0,0,0,114,40,0,0,0,114, - 2,1,0,0,218,11,95,102,105,108,108,95,99,97,99,104, - 101,114,6,0,0,0,114,5,1,0,0,114,88,0,0,0, - 114,4,1,0,0,114,28,0,0,0,114,1,1,0,0,114, - 44,0,0,0,114,255,0,0,0,114,46,0,0,0,114,114, - 0,0,0,114,129,0,0,0,114,154,0,0,0,114,150,0, - 0,0,41,14,114,100,0,0,0,114,119,0,0,0,114,174, - 0,0,0,90,12,105,115,95,110,97,109,101,115,112,97,99, - 101,90,11,116,97,105,108,95,109,111,100,117,108,101,114,126, - 0,0,0,90,5,99,97,99,104,101,90,12,99,97,99,104, - 101,95,109,111,100,117,108,101,90,9,98,97,115,101,95,112, - 97,116,104,114,219,0,0,0,114,159,0,0,0,90,13,105, - 110,105,116,95,102,105,108,101,110,97,109,101,90,9,102,117, - 108,108,95,112,97,116,104,114,158,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,175,0,0,0, - 186,4,0,0,115,70,0,0,0,0,3,6,1,19,1,3, - 1,34,1,13,1,11,1,15,1,10,1,9,2,9,1,9, - 1,15,2,9,1,6,2,12,1,18,1,22,1,10,1,15, - 1,12,1,32,4,12,2,22,1,22,1,22,1,16,1,12, - 1,15,1,14,1,6,1,16,1,18,1,12,1,4,1,122, - 20,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, - 95,115,112,101,99,99,1,0,0,0,0,0,0,0,9,0, - 0,0,13,0,0,0,67,0,0,0,115,11,1,0,0,124, - 0,0,106,0,0,125,1,0,121,31,0,116,1,0,106,2, - 0,124,1,0,112,33,0,116,1,0,106,3,0,131,0,0, - 131,1,0,125,2,0,87,110,33,0,4,116,4,0,116,5, - 0,116,6,0,102,3,0,107,10,0,114,75,0,1,1,1, - 103,0,0,125,2,0,89,110,1,0,88,116,7,0,106,8, - 0,106,9,0,100,1,0,131,1,0,115,112,0,116,10,0, - 124,2,0,131,1,0,124,0,0,95,11,0,110,111,0,116, - 10,0,131,0,0,125,3,0,120,90,0,124,2,0,68,93, - 82,0,125,4,0,124,4,0,106,12,0,100,2,0,131,1, - 0,92,3,0,125,5,0,125,6,0,125,7,0,124,6,0, - 114,191,0,100,3,0,106,13,0,124,5,0,124,7,0,106, - 14,0,131,0,0,131,2,0,125,8,0,110,6,0,124,5, - 0,125,8,0,124,3,0,106,15,0,124,8,0,131,1,0, - 1,113,128,0,87,124,3,0,124,0,0,95,11,0,116,7, - 0,106,8,0,106,9,0,116,16,0,131,1,0,114,7,1, - 100,4,0,100,5,0,132,0,0,124,2,0,68,131,1,0, - 124,0,0,95,17,0,100,6,0,83,41,7,122,68,70,105, - 108,108,32,116,104,101,32,99,97,99,104,101,32,111,102,32, - 112,111,116,101,110,116,105,97,108,32,109,111,100,117,108,101, - 115,32,97,110,100,32,112,97,99,107,97,103,101,115,32,102, - 111,114,32,116,104,105,115,32,100,105,114,101,99,116,111,114, - 121,46,114,0,0,0,0,114,58,0,0,0,122,5,123,125, - 46,123,125,99,1,0,0,0,0,0,0,0,2,0,0,0, - 3,0,0,0,83,0,0,0,115,28,0,0,0,104,0,0, - 124,0,0,93,18,0,125,1,0,124,1,0,106,0,0,131, - 0,0,146,2,0,113,6,0,83,114,4,0,0,0,41,1, - 114,88,0,0,0,41,2,114,22,0,0,0,90,2,102,110, - 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,250, - 9,60,115,101,116,99,111,109,112,62,5,5,0,0,115,2, - 0,0,0,9,0,122,41,70,105,108,101,70,105,110,100,101, - 114,46,95,102,105,108,108,95,99,97,99,104,101,46,60,108, - 111,99,97,108,115,62,46,60,115,101,116,99,111,109,112,62, - 78,41,18,114,35,0,0,0,114,3,0,0,0,90,7,108, - 105,115,116,100,105,114,114,45,0,0,0,114,250,0,0,0, - 218,15,80,101,114,109,105,115,115,105,111,110,69,114,114,111, - 114,218,18,78,111,116,65,68,105,114,101,99,116,111,114,121, - 69,114,114,111,114,114,7,0,0,0,114,8,0,0,0,114, - 9,0,0,0,114,3,1,0,0,114,4,1,0,0,114,83, - 0,0,0,114,47,0,0,0,114,88,0,0,0,218,3,97, - 100,100,114,10,0,0,0,114,5,1,0,0,41,9,114,100, - 0,0,0,114,35,0,0,0,90,8,99,111,110,116,101,110, - 116,115,90,21,108,111,119,101,114,95,115,117,102,102,105,120, - 95,99,111,110,116,101,110,116,115,114,239,0,0,0,114,98, - 0,0,0,114,231,0,0,0,114,219,0,0,0,90,8,110, - 101,119,95,110,97,109,101,114,4,0,0,0,114,4,0,0, - 0,114,5,0,0,0,114,7,1,0,0,232,4,0,0,115, - 34,0,0,0,0,2,9,1,3,1,31,1,22,3,11,3, - 18,1,18,7,9,1,13,1,24,1,6,1,27,2,6,1, - 17,1,9,1,18,1,122,22,70,105,108,101,70,105,110,100, - 101,114,46,95,102,105,108,108,95,99,97,99,104,101,99,1, - 0,0,0,0,0,0,0,3,0,0,0,3,0,0,0,7, - 0,0,0,115,25,0,0,0,135,0,0,135,1,0,102,2, - 0,100,1,0,100,2,0,134,0,0,125,2,0,124,2,0, - 83,41,3,97,20,1,0,0,65,32,99,108,97,115,115,32, - 109,101,116,104,111,100,32,119,104,105,99,104,32,114,101,116, - 117,114,110,115,32,97,32,99,108,111,115,117,114,101,32,116, - 111,32,117,115,101,32,111,110,32,115,121,115,46,112,97,116, - 104,95,104,111,111,107,10,32,32,32,32,32,32,32,32,119, - 104,105,99,104,32,119,105,108,108,32,114,101,116,117,114,110, - 32,97,110,32,105,110,115,116,97,110,99,101,32,117,115,105, - 110,103,32,116,104,101,32,115,112,101,99,105,102,105,101,100, - 32,108,111,97,100,101,114,115,32,97,110,100,32,116,104,101, - 32,112,97,116,104,10,32,32,32,32,32,32,32,32,99,97, - 108,108,101,100,32,111,110,32,116,104,101,32,99,108,111,115, - 117,114,101,46,10,10,32,32,32,32,32,32,32,32,73,102, - 32,116,104,101,32,112,97,116,104,32,99,97,108,108,101,100, - 32,111,110,32,116,104,101,32,99,108,111,115,117,114,101,32, - 105,115,32,110,111,116,32,97,32,100,105,114,101,99,116,111, - 114,121,44,32,73,109,112,111,114,116,69,114,114,111,114,32, - 105,115,10,32,32,32,32,32,32,32,32,114,97,105,115,101, - 100,46,10,10,32,32,32,32,32,32,32,32,99,1,0,0, - 0,0,0,0,0,1,0,0,0,4,0,0,0,19,0,0, - 0,115,43,0,0,0,116,0,0,124,0,0,131,1,0,115, - 30,0,116,1,0,100,1,0,100,2,0,124,0,0,131,1, - 1,130,1,0,136,0,0,124,0,0,136,1,0,140,1,0, - 83,41,3,122,45,80,97,116,104,32,104,111,111,107,32,102, - 111,114,32,105,109,112,111,114,116,108,105,98,46,109,97,99, - 104,105,110,101,114,121,46,70,105,108,101,70,105,110,100,101, - 114,46,122,30,111,110,108,121,32,100,105,114,101,99,116,111, - 114,105,101,115,32,97,114,101,32,115,117,112,112,111,114,116, - 101,100,114,35,0,0,0,41,2,114,46,0,0,0,114,99, - 0,0,0,41,1,114,35,0,0,0,41,2,114,164,0,0, - 0,114,6,1,0,0,114,4,0,0,0,114,5,0,0,0, - 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, - 70,105,108,101,70,105,110,100,101,114,17,5,0,0,115,6, - 0,0,0,0,2,12,1,18,1,122,54,70,105,108,101,70, - 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, - 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, - 111,107,95,102,111,114,95,70,105,108,101,70,105,110,100,101, - 114,114,4,0,0,0,41,3,114,164,0,0,0,114,6,1, - 0,0,114,12,1,0,0,114,4,0,0,0,41,2,114,164, - 0,0,0,114,6,1,0,0,114,5,0,0,0,218,9,112, - 97,116,104,95,104,111,111,107,7,5,0,0,115,4,0,0, - 0,0,10,21,6,122,20,70,105,108,101,70,105,110,100,101, - 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, - 0,0,0,0,1,0,0,0,2,0,0,0,67,0,0,0, - 115,16,0,0,0,100,1,0,106,0,0,124,0,0,106,1, - 0,131,1,0,83,41,2,78,122,16,70,105,108,101,70,105, - 110,100,101,114,40,123,33,114,125,41,41,2,114,47,0,0, - 0,114,35,0,0,0,41,1,114,100,0,0,0,114,4,0, - 0,0,114,4,0,0,0,114,5,0,0,0,114,238,0,0, - 0,25,5,0,0,115,2,0,0,0,0,1,122,19,70,105, - 108,101,70,105,110,100,101,114,46,95,95,114,101,112,114,95, - 95,41,15,114,105,0,0,0,114,104,0,0,0,114,106,0, - 0,0,114,107,0,0,0,114,179,0,0,0,114,244,0,0, - 0,114,123,0,0,0,114,176,0,0,0,114,117,0,0,0, - 114,255,0,0,0,114,175,0,0,0,114,7,1,0,0,114, - 177,0,0,0,114,13,1,0,0,114,238,0,0,0,114,4, + 0,114,5,0,0,0,114,180,0,0,0,245,3,0,0,115, + 0,0,0,0,122,30,95,78,97,109,101,115,112,97,99,101, + 76,111,97,100,101,114,46,99,114,101,97,116,101,95,109,111, + 100,117,108,101,99,2,0,0,0,0,0,0,0,2,0,0, + 0,1,0,0,0,67,0,0,0,115,4,0,0,0,100,0, + 0,83,41,1,78,114,4,0,0,0,41,2,114,100,0,0, + 0,114,184,0,0,0,114,4,0,0,0,114,4,0,0,0, + 114,5,0,0,0,114,185,0,0,0,248,3,0,0,115,2, + 0,0,0,0,1,122,28,95,78,97,109,101,115,112,97,99, + 101,76,111,97,100,101,114,46,101,120,101,99,95,109,111,100, + 117,108,101,99,2,0,0,0,0,0,0,0,2,0,0,0, + 3,0,0,0,67,0,0,0,115,35,0,0,0,116,0,0, + 106,1,0,100,1,0,124,0,0,106,2,0,131,2,0,1, + 116,0,0,106,3,0,124,0,0,124,1,0,131,2,0,83, + 41,2,122,98,76,111,97,100,32,97,32,110,97,109,101,115, + 112,97,99,101,32,109,111,100,117,108,101,46,10,10,32,32, + 32,32,32,32,32,32,84,104,105,115,32,109,101,116,104,111, + 100,32,105,115,32,100,101,112,114,101,99,97,116,101,100,46, + 32,32,85,115,101,32,101,120,101,99,95,109,111,100,117,108, + 101,40,41,32,105,110,115,116,101,97,100,46,10,10,32,32, + 32,32,32,32,32,32,122,38,110,97,109,101,115,112,97,99, + 101,32,109,111,100,117,108,101,32,108,111,97,100,101,100,32, + 119,105,116,104,32,112,97,116,104,32,123,33,114,125,41,4, + 114,114,0,0,0,114,129,0,0,0,114,226,0,0,0,114, + 186,0,0,0,41,2,114,100,0,0,0,114,119,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,114, + 187,0,0,0,251,3,0,0,115,6,0,0,0,0,7,9, + 1,10,1,122,28,95,78,97,109,101,115,112,97,99,101,76, + 111,97,100,101,114,46,108,111,97,100,95,109,111,100,117,108, + 101,78,41,12,114,105,0,0,0,114,104,0,0,0,114,106, + 0,0,0,114,179,0,0,0,114,177,0,0,0,114,244,0, + 0,0,114,153,0,0,0,114,196,0,0,0,114,181,0,0, + 0,114,180,0,0,0,114,185,0,0,0,114,187,0,0,0, + 114,4,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,243,0,0,0,223,3,0,0,115,16,0, + 0,0,12,1,12,3,18,9,12,3,12,3,12,3,12,3, + 12,3,114,243,0,0,0,99,0,0,0,0,0,0,0,0, + 0,0,0,0,5,0,0,0,64,0,0,0,115,160,0,0, + 0,101,0,0,90,1,0,100,0,0,90,2,0,100,1,0, + 90,3,0,101,4,0,100,2,0,100,3,0,132,0,0,131, + 1,0,90,5,0,101,4,0,100,4,0,100,5,0,132,0, + 0,131,1,0,90,6,0,101,4,0,100,6,0,100,7,0, + 132,0,0,131,1,0,90,7,0,101,4,0,100,8,0,100, + 9,0,132,0,0,131,1,0,90,8,0,101,4,0,100,10, + 0,100,11,0,100,12,0,132,1,0,131,1,0,90,9,0, + 101,4,0,100,10,0,100,10,0,100,13,0,100,14,0,132, + 2,0,131,1,0,90,10,0,101,4,0,100,10,0,100,15, + 0,100,16,0,132,1,0,131,1,0,90,11,0,100,10,0, + 83,41,17,218,10,80,97,116,104,70,105,110,100,101,114,122, + 62,77,101,116,97,32,112,97,116,104,32,102,105,110,100,101, + 114,32,102,111,114,32,115,121,115,46,112,97,116,104,32,97, + 110,100,32,112,97,99,107,97,103,101,32,95,95,112,97,116, + 104,95,95,32,97,116,116,114,105,98,117,116,101,115,46,99, + 1,0,0,0,0,0,0,0,2,0,0,0,4,0,0,0, + 67,0,0,0,115,55,0,0,0,120,48,0,116,0,0,106, + 1,0,106,2,0,131,0,0,68,93,31,0,125,1,0,116, + 3,0,124,1,0,100,1,0,131,2,0,114,16,0,124,1, + 0,106,4,0,131,0,0,1,113,16,0,87,100,2,0,83, + 41,3,122,125,67,97,108,108,32,116,104,101,32,105,110,118, + 97,108,105,100,97,116,101,95,99,97,99,104,101,115,40,41, + 32,109,101,116,104,111,100,32,111,110,32,97,108,108,32,112, + 97,116,104,32,101,110,116,114,121,32,102,105,110,100,101,114, + 115,10,32,32,32,32,32,32,32,32,115,116,111,114,101,100, + 32,105,110,32,115,121,115,46,112,97,116,104,95,105,109,112, + 111,114,116,101,114,95,99,97,99,104,101,115,32,40,119,104, + 101,114,101,32,105,109,112,108,101,109,101,110,116,101,100,41, + 46,218,17,105,110,118,97,108,105,100,97,116,101,95,99,97, + 99,104,101,115,78,41,5,114,7,0,0,0,218,19,112,97, + 116,104,95,105,109,112,111,114,116,101,114,95,99,97,99,104, + 101,218,6,118,97,108,117,101,115,114,108,0,0,0,114,246, + 0,0,0,41,2,114,164,0,0,0,218,6,102,105,110,100, + 101,114,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,114,246,0,0,0,13,4,0,0,115,6,0,0,0,0, + 4,22,1,15,1,122,28,80,97,116,104,70,105,110,100,101, + 114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99, + 104,101,115,99,2,0,0,0,0,0,0,0,3,0,0,0, + 12,0,0,0,67,0,0,0,115,107,0,0,0,116,0,0, + 106,1,0,100,1,0,107,9,0,114,41,0,116,0,0,106, + 1,0,12,114,41,0,116,2,0,106,3,0,100,2,0,116, + 4,0,131,2,0,1,120,59,0,116,0,0,106,1,0,68, + 93,44,0,125,2,0,121,14,0,124,2,0,124,1,0,131, + 1,0,83,87,113,51,0,4,116,5,0,107,10,0,114,94, + 0,1,1,1,119,51,0,89,113,51,0,88,113,51,0,87, + 100,1,0,83,100,1,0,83,41,3,122,113,83,101,97,114, + 99,104,32,115,101,113,117,101,110,99,101,32,111,102,32,104, + 111,111,107,115,32,102,111,114,32,97,32,102,105,110,100,101, + 114,32,102,111,114,32,39,112,97,116,104,39,46,10,10,32, + 32,32,32,32,32,32,32,73,102,32,39,104,111,111,107,115, + 39,32,105,115,32,102,97,108,115,101,32,116,104,101,110,32, + 117,115,101,32,115,121,115,46,112,97,116,104,95,104,111,111, + 107,115,46,10,10,32,32,32,32,32,32,32,32,78,122,23, + 115,121,115,46,112,97,116,104,95,104,111,111,107,115,32,105, + 115,32,101,109,112,116,121,41,6,114,7,0,0,0,218,10, + 112,97,116,104,95,104,111,111,107,115,114,60,0,0,0,114, + 61,0,0,0,114,118,0,0,0,114,99,0,0,0,41,3, + 114,164,0,0,0,114,35,0,0,0,90,4,104,111,111,107, + 114,4,0,0,0,114,4,0,0,0,114,5,0,0,0,218, + 11,95,112,97,116,104,95,104,111,111,107,115,21,4,0,0, + 115,16,0,0,0,0,7,25,1,16,1,16,1,3,1,14, + 1,13,1,12,2,122,22,80,97,116,104,70,105,110,100,101, + 114,46,95,112,97,116,104,95,104,111,111,107,115,99,2,0, + 0,0,0,0,0,0,3,0,0,0,19,0,0,0,67,0, + 0,0,115,123,0,0,0,124,1,0,100,1,0,107,2,0, + 114,53,0,121,16,0,116,0,0,106,1,0,131,0,0,125, + 1,0,87,110,22,0,4,116,2,0,107,10,0,114,52,0, + 1,1,1,100,2,0,83,89,110,1,0,88,121,17,0,116, + 3,0,106,4,0,124,1,0,25,125,2,0,87,110,46,0, + 4,116,5,0,107,10,0,114,118,0,1,1,1,124,0,0, + 106,6,0,124,1,0,131,1,0,125,2,0,124,2,0,116, + 3,0,106,4,0,124,1,0,60,89,110,1,0,88,124,2, + 0,83,41,3,122,210,71,101,116,32,116,104,101,32,102,105, + 110,100,101,114,32,102,111,114,32,116,104,101,32,112,97,116, + 104,32,101,110,116,114,121,32,102,114,111,109,32,115,121,115, + 46,112,97,116,104,95,105,109,112,111,114,116,101,114,95,99, + 97,99,104,101,46,10,10,32,32,32,32,32,32,32,32,73, + 102,32,116,104,101,32,112,97,116,104,32,101,110,116,114,121, + 32,105,115,32,110,111,116,32,105,110,32,116,104,101,32,99, + 97,99,104,101,44,32,102,105,110,100,32,116,104,101,32,97, + 112,112,114,111,112,114,105,97,116,101,32,102,105,110,100,101, + 114,10,32,32,32,32,32,32,32,32,97,110,100,32,99,97, + 99,104,101,32,105,116,46,32,73,102,32,110,111,32,102,105, + 110,100,101,114,32,105,115,32,97,118,97,105,108,97,98,108, + 101,44,32,115,116,111,114,101,32,78,111,110,101,46,10,10, + 32,32,32,32,32,32,32,32,114,30,0,0,0,78,41,7, + 114,3,0,0,0,114,45,0,0,0,218,17,70,105,108,101, + 78,111,116,70,111,117,110,100,69,114,114,111,114,114,7,0, + 0,0,114,247,0,0,0,114,131,0,0,0,114,251,0,0, + 0,41,3,114,164,0,0,0,114,35,0,0,0,114,249,0, + 0,0,114,4,0,0,0,114,4,0,0,0,114,5,0,0, + 0,218,20,95,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,38,4,0,0,115,22,0,0,0, + 0,8,12,1,3,1,16,1,13,3,9,1,3,1,17,1, + 13,1,15,1,18,1,122,31,80,97,116,104,70,105,110,100, + 101,114,46,95,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,99,3,0,0,0,0,0,0,0, + 6,0,0,0,3,0,0,0,67,0,0,0,115,119,0,0, + 0,116,0,0,124,2,0,100,1,0,131,2,0,114,39,0, + 124,2,0,106,1,0,124,1,0,131,1,0,92,2,0,125, + 3,0,125,4,0,110,21,0,124,2,0,106,2,0,124,1, + 0,131,1,0,125,3,0,103,0,0,125,4,0,124,3,0, + 100,0,0,107,9,0,114,88,0,116,3,0,106,4,0,124, + 1,0,124,3,0,131,2,0,83,116,3,0,106,5,0,124, + 1,0,100,0,0,131,2,0,125,5,0,124,4,0,124,5, + 0,95,6,0,124,5,0,83,41,2,78,114,117,0,0,0, + 41,7,114,108,0,0,0,114,117,0,0,0,114,176,0,0, + 0,114,114,0,0,0,114,173,0,0,0,114,154,0,0,0, + 114,150,0,0,0,41,6,114,164,0,0,0,114,119,0,0, + 0,114,249,0,0,0,114,120,0,0,0,114,121,0,0,0, + 114,158,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,218,16,95,108,101,103,97,99,121,95,103,101, + 116,95,115,112,101,99,60,4,0,0,115,18,0,0,0,0, + 4,15,1,24,2,15,1,6,1,12,1,16,1,18,1,9, + 1,122,27,80,97,116,104,70,105,110,100,101,114,46,95,108, + 101,103,97,99,121,95,103,101,116,95,115,112,101,99,78,99, + 4,0,0,0,0,0,0,0,9,0,0,0,5,0,0,0, + 67,0,0,0,115,243,0,0,0,103,0,0,125,4,0,120, + 230,0,124,2,0,68,93,191,0,125,5,0,116,0,0,124, + 5,0,116,1,0,116,2,0,102,2,0,131,2,0,115,43, + 0,113,13,0,124,0,0,106,3,0,124,5,0,131,1,0, + 125,6,0,124,6,0,100,1,0,107,9,0,114,13,0,116, + 4,0,124,6,0,100,2,0,131,2,0,114,106,0,124,6, + 0,106,5,0,124,1,0,124,3,0,131,2,0,125,7,0, + 110,18,0,124,0,0,106,6,0,124,1,0,124,6,0,131, + 2,0,125,7,0,124,7,0,100,1,0,107,8,0,114,139, + 0,113,13,0,124,7,0,106,7,0,100,1,0,107,9,0, + 114,158,0,124,7,0,83,124,7,0,106,8,0,125,8,0, + 124,8,0,100,1,0,107,8,0,114,191,0,116,9,0,100, + 3,0,131,1,0,130,1,0,124,4,0,106,10,0,124,8, + 0,131,1,0,1,113,13,0,87,116,11,0,106,12,0,124, + 1,0,100,1,0,131,2,0,125,7,0,124,4,0,124,7, + 0,95,8,0,124,7,0,83,100,1,0,83,41,4,122,63, + 70,105,110,100,32,116,104,101,32,108,111,97,100,101,114,32, + 111,114,32,110,97,109,101,115,112,97,99,101,95,112,97,116, + 104,32,102,111,114,32,116,104,105,115,32,109,111,100,117,108, + 101,47,112,97,99,107,97,103,101,32,110,97,109,101,46,78, + 114,175,0,0,0,122,19,115,112,101,99,32,109,105,115,115, + 105,110,103,32,108,111,97,100,101,114,41,13,114,137,0,0, + 0,114,69,0,0,0,218,5,98,121,116,101,115,114,253,0, + 0,0,114,108,0,0,0,114,175,0,0,0,114,254,0,0, + 0,114,120,0,0,0,114,150,0,0,0,114,99,0,0,0, + 114,143,0,0,0,114,114,0,0,0,114,154,0,0,0,41, + 9,114,164,0,0,0,114,119,0,0,0,114,35,0,0,0, + 114,174,0,0,0,218,14,110,97,109,101,115,112,97,99,101, + 95,112,97,116,104,90,5,101,110,116,114,121,114,249,0,0, + 0,114,158,0,0,0,114,121,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,218,9,95,103,101,116, + 95,115,112,101,99,75,4,0,0,115,40,0,0,0,0,5, + 6,1,13,1,21,1,3,1,15,1,12,1,15,1,21,2, + 18,1,12,1,3,1,15,1,4,1,9,1,12,1,12,5, + 17,2,18,1,9,1,122,20,80,97,116,104,70,105,110,100, + 101,114,46,95,103,101,116,95,115,112,101,99,99,4,0,0, + 0,0,0,0,0,6,0,0,0,4,0,0,0,67,0,0, + 0,115,140,0,0,0,124,2,0,100,1,0,107,8,0,114, + 21,0,116,0,0,106,1,0,125,2,0,124,0,0,106,2, + 0,124,1,0,124,2,0,124,3,0,131,3,0,125,4,0, + 124,4,0,100,1,0,107,8,0,114,58,0,100,1,0,83, + 124,4,0,106,3,0,100,1,0,107,8,0,114,132,0,124, + 4,0,106,4,0,125,5,0,124,5,0,114,125,0,100,2, + 0,124,4,0,95,5,0,116,6,0,124,1,0,124,5,0, + 124,0,0,106,2,0,131,3,0,124,4,0,95,4,0,124, + 4,0,83,100,1,0,83,110,4,0,124,4,0,83,100,1, + 0,83,41,3,122,98,102,105,110,100,32,116,104,101,32,109, + 111,100,117,108,101,32,111,110,32,115,121,115,46,112,97,116, + 104,32,111,114,32,39,112,97,116,104,39,32,98,97,115,101, + 100,32,111,110,32,115,121,115,46,112,97,116,104,95,104,111, + 111,107,115,32,97,110,100,10,32,32,32,32,32,32,32,32, + 115,121,115,46,112,97,116,104,95,105,109,112,111,114,116,101, + 114,95,99,97,99,104,101,46,78,90,9,110,97,109,101,115, + 112,97,99,101,41,7,114,7,0,0,0,114,35,0,0,0, + 114,1,1,0,0,114,120,0,0,0,114,150,0,0,0,114, + 152,0,0,0,114,224,0,0,0,41,6,114,164,0,0,0, + 114,119,0,0,0,114,35,0,0,0,114,174,0,0,0,114, + 158,0,0,0,114,0,1,0,0,114,4,0,0,0,114,4, + 0,0,0,114,5,0,0,0,114,175,0,0,0,107,4,0, + 0,115,26,0,0,0,0,4,12,1,9,1,21,1,12,1, + 4,1,15,1,9,1,6,3,9,1,24,1,4,2,7,2, + 122,20,80,97,116,104,70,105,110,100,101,114,46,102,105,110, + 100,95,115,112,101,99,99,3,0,0,0,0,0,0,0,4, + 0,0,0,3,0,0,0,67,0,0,0,115,41,0,0,0, + 124,0,0,106,0,0,124,1,0,124,2,0,131,2,0,125, + 3,0,124,3,0,100,1,0,107,8,0,114,34,0,100,1, + 0,83,124,3,0,106,1,0,83,41,2,122,170,102,105,110, + 100,32,116,104,101,32,109,111,100,117,108,101,32,111,110,32, + 115,121,115,46,112,97,116,104,32,111,114,32,39,112,97,116, + 104,39,32,98,97,115,101,100,32,111,110,32,115,121,115,46, + 112,97,116,104,95,104,111,111,107,115,32,97,110,100,10,32, + 32,32,32,32,32,32,32,115,121,115,46,112,97,116,104,95, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,46,10, + 10,32,32,32,32,32,32,32,32,84,104,105,115,32,109,101, + 116,104,111,100,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,46,32,32,85,115,101,32,102,105,110,100,95,115,112, + 101,99,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,78,41,2,114,175,0,0,0,114, + 120,0,0,0,41,4,114,164,0,0,0,114,119,0,0,0, + 114,35,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,176,0,0,0,129,4, + 0,0,115,8,0,0,0,0,8,18,1,12,1,4,1,122, + 22,80,97,116,104,70,105,110,100,101,114,46,102,105,110,100, + 95,109,111,100,117,108,101,41,12,114,105,0,0,0,114,104, + 0,0,0,114,106,0,0,0,114,107,0,0,0,114,177,0, + 0,0,114,246,0,0,0,114,251,0,0,0,114,253,0,0, + 0,114,254,0,0,0,114,1,1,0,0,114,175,0,0,0, + 114,176,0,0,0,114,4,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,245,0,0,0,9,4, + 0,0,115,22,0,0,0,12,2,6,2,18,8,18,17,18, + 22,18,15,3,1,18,31,3,1,21,21,3,1,114,245,0, + 0,0,99,0,0,0,0,0,0,0,0,0,0,0,0,3, + 0,0,0,64,0,0,0,115,133,0,0,0,101,0,0,90, + 1,0,100,0,0,90,2,0,100,1,0,90,3,0,100,2, + 0,100,3,0,132,0,0,90,4,0,100,4,0,100,5,0, + 132,0,0,90,5,0,101,6,0,90,7,0,100,6,0,100, + 7,0,132,0,0,90,8,0,100,8,0,100,9,0,132,0, + 0,90,9,0,100,10,0,100,11,0,100,12,0,132,1,0, + 90,10,0,100,13,0,100,14,0,132,0,0,90,11,0,101, + 12,0,100,15,0,100,16,0,132,0,0,131,1,0,90,13, + 0,100,17,0,100,18,0,132,0,0,90,14,0,100,10,0, + 83,41,19,218,10,70,105,108,101,70,105,110,100,101,114,122, + 172,70,105,108,101,45,98,97,115,101,100,32,102,105,110,100, + 101,114,46,10,10,32,32,32,32,73,110,116,101,114,97,99, + 116,105,111,110,115,32,119,105,116,104,32,116,104,101,32,102, + 105,108,101,32,115,121,115,116,101,109,32,97,114,101,32,99, + 97,99,104,101,100,32,102,111,114,32,112,101,114,102,111,114, + 109,97,110,99,101,44,32,98,101,105,110,103,10,32,32,32, + 32,114,101,102,114,101,115,104,101,100,32,119,104,101,110,32, + 116,104,101,32,100,105,114,101,99,116,111,114,121,32,116,104, + 101,32,102,105,110,100,101,114,32,105,115,32,104,97,110,100, + 108,105,110,103,32,104,97,115,32,98,101,101,110,32,109,111, + 100,105,102,105,101,100,46,10,10,32,32,32,32,99,2,0, + 0,0,0,0,0,0,5,0,0,0,5,0,0,0,7,0, + 0,0,115,122,0,0,0,103,0,0,125,3,0,120,52,0, + 124,2,0,68,93,44,0,92,2,0,137,0,0,125,4,0, + 124,3,0,106,0,0,135,0,0,102,1,0,100,1,0,100, + 2,0,134,0,0,124,4,0,68,131,1,0,131,1,0,1, + 113,13,0,87,124,3,0,124,0,0,95,1,0,124,1,0, + 112,79,0,100,3,0,124,0,0,95,2,0,100,6,0,124, + 0,0,95,3,0,116,4,0,131,0,0,124,0,0,95,5, + 0,116,4,0,131,0,0,124,0,0,95,6,0,100,5,0, + 83,41,7,122,154,73,110,105,116,105,97,108,105,122,101,32, + 119,105,116,104,32,116,104,101,32,112,97,116,104,32,116,111, + 32,115,101,97,114,99,104,32,111,110,32,97,110,100,32,97, + 32,118,97,114,105,97,98,108,101,32,110,117,109,98,101,114, + 32,111,102,10,32,32,32,32,32,32,32,32,50,45,116,117, + 112,108,101,115,32,99,111,110,116,97,105,110,105,110,103,32, + 116,104,101,32,108,111,97,100,101,114,32,97,110,100,32,116, + 104,101,32,102,105,108,101,32,115,117,102,102,105,120,101,115, + 32,116,104,101,32,108,111,97,100,101,114,10,32,32,32,32, + 32,32,32,32,114,101,99,111,103,110,105,122,101,115,46,99, + 1,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, + 51,0,0,0,115,27,0,0,0,124,0,0,93,17,0,125, + 1,0,124,1,0,136,0,0,102,2,0,86,1,113,3,0, + 100,0,0,83,41,1,78,114,4,0,0,0,41,2,114,22, + 0,0,0,114,219,0,0,0,41,1,114,120,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,221,0,0,0,158,4, + 0,0,115,2,0,0,0,6,0,122,38,70,105,108,101,70, + 105,110,100,101,114,46,95,95,105,110,105,116,95,95,46,60, + 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, + 62,114,58,0,0,0,114,29,0,0,0,78,114,87,0,0, + 0,41,7,114,143,0,0,0,218,8,95,108,111,97,100,101, + 114,115,114,35,0,0,0,218,11,95,112,97,116,104,95,109, + 116,105,109,101,218,3,115,101,116,218,11,95,112,97,116,104, + 95,99,97,99,104,101,218,19,95,114,101,108,97,120,101,100, + 95,112,97,116,104,95,99,97,99,104,101,41,5,114,100,0, + 0,0,114,35,0,0,0,218,14,108,111,97,100,101,114,95, + 100,101,116,97,105,108,115,90,7,108,111,97,100,101,114,115, + 114,160,0,0,0,114,4,0,0,0,41,1,114,120,0,0, + 0,114,5,0,0,0,114,179,0,0,0,152,4,0,0,115, + 16,0,0,0,0,4,6,1,19,1,36,1,9,2,15,1, + 9,1,12,1,122,19,70,105,108,101,70,105,110,100,101,114, + 46,95,95,105,110,105,116,95,95,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,13, + 0,0,0,100,3,0,124,0,0,95,0,0,100,2,0,83, + 41,4,122,31,73,110,118,97,108,105,100,97,116,101,32,116, + 104,101,32,100,105,114,101,99,116,111,114,121,32,109,116,105, + 109,101,46,114,29,0,0,0,78,114,87,0,0,0,41,1, + 114,4,1,0,0,41,1,114,100,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,5,0,0,0,114,246,0,0,0, + 166,4,0,0,115,2,0,0,0,0,2,122,28,70,105,108, + 101,70,105,110,100,101,114,46,105,110,118,97,108,105,100,97, + 116,101,95,99,97,99,104,101,115,99,2,0,0,0,0,0, + 0,0,3,0,0,0,2,0,0,0,67,0,0,0,115,59, + 0,0,0,124,0,0,106,0,0,124,1,0,131,1,0,125, + 2,0,124,2,0,100,1,0,107,8,0,114,37,0,100,1, + 0,103,0,0,102,2,0,83,124,2,0,106,1,0,124,2, + 0,106,2,0,112,55,0,103,0,0,102,2,0,83,41,2, + 122,197,84,114,121,32,116,111,32,102,105,110,100,32,97,32, + 108,111,97,100,101,114,32,102,111,114,32,116,104,101,32,115, + 112,101,99,105,102,105,101,100,32,109,111,100,117,108,101,44, + 32,111,114,32,116,104,101,32,110,97,109,101,115,112,97,99, + 101,10,32,32,32,32,32,32,32,32,112,97,99,107,97,103, + 101,32,112,111,114,116,105,111,110,115,46,32,82,101,116,117, + 114,110,115,32,40,108,111,97,100,101,114,44,32,108,105,115, + 116,45,111,102,45,112,111,114,116,105,111,110,115,41,46,10, + 10,32,32,32,32,32,32,32,32,84,104,105,115,32,109,101, + 116,104,111,100,32,105,115,32,100,101,112,114,101,99,97,116, + 101,100,46,32,32,85,115,101,32,102,105,110,100,95,115,112, + 101,99,40,41,32,105,110,115,116,101,97,100,46,10,10,32, + 32,32,32,32,32,32,32,78,41,3,114,175,0,0,0,114, + 120,0,0,0,114,150,0,0,0,41,3,114,100,0,0,0, + 114,119,0,0,0,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,117,0,0,0,172,4, + 0,0,115,8,0,0,0,0,7,15,1,12,1,10,1,122, + 22,70,105,108,101,70,105,110,100,101,114,46,102,105,110,100, + 95,108,111,97,100,101,114,99,6,0,0,0,0,0,0,0, + 7,0,0,0,7,0,0,0,67,0,0,0,115,40,0,0, + 0,124,1,0,124,2,0,124,3,0,131,2,0,125,6,0, + 116,0,0,124,2,0,124,3,0,100,1,0,124,6,0,100, + 2,0,124,4,0,131,2,2,83,41,3,78,114,120,0,0, + 0,114,150,0,0,0,41,1,114,161,0,0,0,41,7,114, + 100,0,0,0,114,159,0,0,0,114,119,0,0,0,114,35, + 0,0,0,90,4,115,109,115,108,114,174,0,0,0,114,120, 0,0,0,114,4,0,0,0,114,4,0,0,0,114,5,0, - 0,0,114,0,1,0,0,140,4,0,0,115,20,0,0,0, - 12,7,6,2,12,14,12,4,6,2,12,12,12,5,15,46, - 12,31,18,18,114,0,1,0,0,99,4,0,0,0,0,0, - 0,0,6,0,0,0,11,0,0,0,67,0,0,0,115,195, - 0,0,0,124,0,0,106,0,0,100,1,0,131,1,0,125, - 4,0,124,0,0,106,0,0,100,2,0,131,1,0,125,5, - 0,124,4,0,115,99,0,124,5,0,114,54,0,124,5,0, - 106,1,0,125,4,0,110,45,0,124,2,0,124,3,0,107, - 2,0,114,84,0,116,2,0,124,1,0,124,2,0,131,2, - 0,125,4,0,110,15,0,116,3,0,124,1,0,124,2,0, - 131,2,0,125,4,0,124,5,0,115,126,0,116,4,0,124, - 1,0,124,2,0,100,3,0,124,4,0,131,2,1,125,5, - 0,121,44,0,124,5,0,124,0,0,100,2,0,60,124,4, - 0,124,0,0,100,1,0,60,124,2,0,124,0,0,100,4, - 0,60,124,3,0,124,0,0,100,5,0,60,87,110,18,0, - 4,116,5,0,107,10,0,114,190,0,1,1,1,89,110,1, - 0,88,100,0,0,83,41,6,78,218,10,95,95,108,111,97, - 100,101,114,95,95,218,8,95,95,115,112,101,99,95,95,114, - 120,0,0,0,90,8,95,95,102,105,108,101,95,95,90,10, - 95,95,99,97,99,104,101,100,95,95,41,6,218,3,103,101, - 116,114,120,0,0,0,114,217,0,0,0,114,212,0,0,0, - 114,161,0,0,0,218,9,69,120,99,101,112,116,105,111,110, - 41,6,90,2,110,115,114,98,0,0,0,90,8,112,97,116, - 104,110,97,109,101,90,9,99,112,97,116,104,110,97,109,101, - 114,120,0,0,0,114,158,0,0,0,114,4,0,0,0,114, - 4,0,0,0,114,5,0,0,0,218,14,95,102,105,120,95, - 117,112,95,109,111,100,117,108,101,31,5,0,0,115,34,0, - 0,0,0,2,15,1,15,1,6,1,6,1,12,1,12,1, - 18,2,15,1,6,1,21,1,3,1,10,1,10,1,10,1, - 14,1,13,2,114,18,1,0,0,99,0,0,0,0,0,0, - 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,55, - 0,0,0,116,0,0,116,1,0,106,2,0,131,0,0,102, - 2,0,125,0,0,116,3,0,116,4,0,102,2,0,125,1, - 0,116,5,0,116,6,0,102,2,0,125,2,0,124,0,0, - 124,1,0,124,2,0,103,3,0,83,41,1,122,95,82,101, - 116,117,114,110,115,32,97,32,108,105,115,116,32,111,102,32, - 102,105,108,101,45,98,97,115,101,100,32,109,111,100,117,108, - 101,32,108,111,97,100,101,114,115,46,10,10,32,32,32,32, - 69,97,99,104,32,105,116,101,109,32,105,115,32,97,32,116, - 117,112,108,101,32,40,108,111,97,100,101,114,44,32,115,117, - 102,102,105,120,101,115,41,46,10,32,32,32,32,41,7,114, - 218,0,0,0,114,139,0,0,0,218,18,101,120,116,101,110, - 115,105,111,110,95,115,117,102,102,105,120,101,115,114,212,0, - 0,0,114,84,0,0,0,114,217,0,0,0,114,74,0,0, - 0,41,3,90,10,101,120,116,101,110,115,105,111,110,115,90, - 6,115,111,117,114,99,101,90,8,98,121,116,101,99,111,100, - 101,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, - 114,155,0,0,0,54,5,0,0,115,8,0,0,0,0,5, - 18,1,12,1,12,1,114,155,0,0,0,99,1,0,0,0, - 0,0,0,0,12,0,0,0,12,0,0,0,67,0,0,0, - 115,70,2,0,0,124,0,0,97,0,0,116,0,0,106,1, - 0,97,1,0,116,0,0,106,2,0,97,2,0,116,1,0, - 106,3,0,116,4,0,25,125,1,0,120,76,0,100,26,0, - 68,93,68,0,125,2,0,124,2,0,116,1,0,106,3,0, - 107,7,0,114,83,0,116,0,0,106,5,0,124,2,0,131, - 1,0,125,3,0,110,13,0,116,1,0,106,3,0,124,2, - 0,25,125,3,0,116,6,0,124,1,0,124,2,0,124,3, - 0,131,3,0,1,113,44,0,87,100,5,0,100,6,0,103, - 1,0,102,2,0,100,7,0,100,8,0,100,6,0,103,2, - 0,102,2,0,102,2,0,125,4,0,120,149,0,124,4,0, - 68,93,129,0,92,2,0,125,5,0,125,6,0,116,7,0, - 100,9,0,100,10,0,132,0,0,124,6,0,68,131,1,0, - 131,1,0,115,199,0,116,8,0,130,1,0,124,6,0,100, - 11,0,25,125,7,0,124,5,0,116,1,0,106,3,0,107, - 6,0,114,241,0,116,1,0,106,3,0,124,5,0,25,125, - 8,0,80,113,156,0,121,20,0,116,0,0,106,5,0,124, - 5,0,131,1,0,125,8,0,80,87,113,156,0,4,116,9, - 0,107,10,0,114,28,1,1,1,1,119,156,0,89,113,156, - 0,88,113,156,0,87,116,9,0,100,12,0,131,1,0,130, - 1,0,116,6,0,124,1,0,100,13,0,124,8,0,131,3, - 0,1,116,6,0,124,1,0,100,14,0,124,7,0,131,3, - 0,1,116,6,0,124,1,0,100,15,0,100,16,0,106,10, - 0,124,6,0,131,1,0,131,3,0,1,121,19,0,116,0, - 0,106,5,0,100,17,0,131,1,0,125,9,0,87,110,24, - 0,4,116,9,0,107,10,0,114,147,1,1,1,1,100,18, - 0,125,9,0,89,110,1,0,88,116,6,0,124,1,0,100, - 17,0,124,9,0,131,3,0,1,116,0,0,106,5,0,100, - 19,0,131,1,0,125,10,0,116,6,0,124,1,0,100,19, - 0,124,10,0,131,3,0,1,124,5,0,100,7,0,107,2, - 0,114,238,1,116,0,0,106,5,0,100,20,0,131,1,0, - 125,11,0,116,6,0,124,1,0,100,21,0,124,11,0,131, - 3,0,1,116,6,0,124,1,0,100,22,0,116,11,0,131, - 0,0,131,3,0,1,116,12,0,106,13,0,116,2,0,106, - 14,0,131,0,0,131,1,0,1,124,5,0,100,7,0,107, - 2,0,114,66,2,116,15,0,106,16,0,100,23,0,131,1, - 0,1,100,24,0,116,12,0,107,6,0,114,66,2,100,25, - 0,116,17,0,95,18,0,100,18,0,83,41,27,122,205,83, - 101,116,117,112,32,116,104,101,32,112,97,116,104,45,98,97, - 115,101,100,32,105,109,112,111,114,116,101,114,115,32,102,111, - 114,32,105,109,112,111,114,116,108,105,98,32,98,121,32,105, - 109,112,111,114,116,105,110,103,32,110,101,101,100,101,100,10, - 32,32,32,32,98,117,105,108,116,45,105,110,32,109,111,100, - 117,108,101,115,32,97,110,100,32,105,110,106,101,99,116,105, - 110,103,32,116,104,101,109,32,105,110,116,111,32,116,104,101, - 32,103,108,111,98,97,108,32,110,97,109,101,115,112,97,99, - 101,46,10,10,32,32,32,32,79,116,104,101,114,32,99,111, - 109,112,111,110,101,110,116,115,32,97,114,101,32,101,120,116, - 114,97,99,116,101,100,32,102,114,111,109,32,116,104,101,32, - 99,111,114,101,32,98,111,111,116,115,116,114,97,112,32,109, - 111,100,117,108,101,46,10,10,32,32,32,32,114,49,0,0, - 0,114,60,0,0,0,218,8,98,117,105,108,116,105,110,115, - 114,136,0,0,0,90,5,112,111,115,105,120,250,1,47,218, - 2,110,116,250,1,92,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,115,0,0,0,115,33,0,0,0, - 124,0,0,93,23,0,125,1,0,116,0,0,124,1,0,131, - 1,0,100,0,0,107,2,0,86,1,113,3,0,100,1,0, - 83,41,2,114,29,0,0,0,78,41,1,114,31,0,0,0, - 41,2,114,22,0,0,0,114,77,0,0,0,114,4,0,0, - 0,114,4,0,0,0,114,5,0,0,0,114,221,0,0,0, - 90,5,0,0,115,2,0,0,0,6,0,122,25,95,115,101, - 116,117,112,46,60,108,111,99,97,108,115,62,46,60,103,101, - 110,101,120,112,114,62,114,59,0,0,0,122,30,105,109,112, - 111,114,116,108,105,98,32,114,101,113,117,105,114,101,115,32, - 112,111,115,105,120,32,111,114,32,110,116,114,3,0,0,0, - 114,25,0,0,0,114,21,0,0,0,114,30,0,0,0,90, - 7,95,116,104,114,101,97,100,78,90,8,95,119,101,97,107, - 114,101,102,90,6,119,105,110,114,101,103,114,163,0,0,0, - 114,6,0,0,0,122,4,46,112,121,119,122,6,95,100,46, - 112,121,100,84,41,4,122,3,95,105,111,122,9,95,119,97, - 114,110,105,110,103,115,122,8,98,117,105,108,116,105,110,115, - 122,7,109,97,114,115,104,97,108,41,19,114,114,0,0,0, - 114,7,0,0,0,114,139,0,0,0,114,233,0,0,0,114, - 105,0,0,0,90,18,95,98,117,105,108,116,105,110,95,102, - 114,111,109,95,110,97,109,101,114,109,0,0,0,218,3,97, - 108,108,218,14,65,115,115,101,114,116,105,111,110,69,114,114, - 111,114,114,99,0,0,0,114,26,0,0,0,114,11,0,0, - 0,114,223,0,0,0,114,143,0,0,0,114,19,1,0,0, - 114,84,0,0,0,114,157,0,0,0,114,162,0,0,0,114, - 167,0,0,0,41,12,218,17,95,98,111,111,116,115,116,114, - 97,112,95,109,111,100,117,108,101,90,11,115,101,108,102,95, - 109,111,100,117,108,101,90,12,98,117,105,108,116,105,110,95, - 110,97,109,101,90,14,98,117,105,108,116,105,110,95,109,111, - 100,117,108,101,90,10,111,115,95,100,101,116,97,105,108,115, - 90,10,98,117,105,108,116,105,110,95,111,115,114,21,0,0, - 0,114,25,0,0,0,90,9,111,115,95,109,111,100,117,108, - 101,90,13,116,104,114,101,97,100,95,109,111,100,117,108,101, - 90,14,119,101,97,107,114,101,102,95,109,111,100,117,108,101, - 90,13,119,105,110,114,101,103,95,109,111,100,117,108,101,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,6, - 95,115,101,116,117,112,65,5,0,0,115,82,0,0,0,0, - 8,6,1,9,1,9,3,13,1,13,1,15,1,18,2,13, - 1,20,3,33,1,19,2,31,1,10,1,15,1,13,1,4, - 2,3,1,15,1,5,1,13,1,12,2,12,1,16,1,16, - 1,25,3,3,1,19,1,13,2,11,1,16,3,15,1,16, - 3,12,1,15,1,16,3,19,1,19,1,12,1,13,1,12, - 1,114,27,1,0,0,99,1,0,0,0,0,0,0,0,2, - 0,0,0,3,0,0,0,67,0,0,0,115,116,0,0,0, - 116,0,0,124,0,0,131,1,0,1,116,1,0,131,0,0, - 125,1,0,116,2,0,106,3,0,106,4,0,116,5,0,106, - 6,0,124,1,0,140,0,0,103,1,0,131,1,0,1,116, - 7,0,106,8,0,100,1,0,107,2,0,114,78,0,116,2, - 0,106,9,0,106,10,0,116,11,0,131,1,0,1,116,2, - 0,106,9,0,106,10,0,116,12,0,131,1,0,1,116,5, - 0,124,0,0,95,5,0,116,13,0,124,0,0,95,13,0, - 100,2,0,83,41,3,122,41,73,110,115,116,97,108,108,32, - 116,104,101,32,112,97,116,104,45,98,97,115,101,100,32,105, - 109,112,111,114,116,32,99,111,109,112,111,110,101,110,116,115, - 46,114,22,1,0,0,78,41,14,114,27,1,0,0,114,155, - 0,0,0,114,7,0,0,0,114,248,0,0,0,114,143,0, - 0,0,114,0,1,0,0,114,13,1,0,0,114,3,0,0, - 0,114,105,0,0,0,218,9,109,101,116,97,95,112,97,116, - 104,114,157,0,0,0,114,162,0,0,0,114,243,0,0,0, - 114,212,0,0,0,41,2,114,26,1,0,0,90,17,115,117, - 112,112,111,114,116,101,100,95,108,111,97,100,101,114,115,114, - 4,0,0,0,114,4,0,0,0,114,5,0,0,0,218,8, - 95,105,110,115,116,97,108,108,133,5,0,0,115,16,0,0, - 0,0,2,10,1,9,1,28,1,15,1,16,1,16,4,9, - 1,114,29,1,0,0,41,3,122,3,119,105,110,114,1,0, - 0,0,114,2,0,0,0,41,56,114,107,0,0,0,114,10, - 0,0,0,114,11,0,0,0,114,17,0,0,0,114,19,0, - 0,0,114,28,0,0,0,114,38,0,0,0,114,39,0,0, - 0,114,43,0,0,0,114,44,0,0,0,114,46,0,0,0, - 114,55,0,0,0,218,4,116,121,112,101,218,8,95,95,99, - 111,100,101,95,95,114,138,0,0,0,114,15,0,0,0,114, - 128,0,0,0,114,14,0,0,0,114,18,0,0,0,90,17, - 95,82,65,87,95,77,65,71,73,67,95,78,85,77,66,69, - 82,114,73,0,0,0,114,72,0,0,0,114,84,0,0,0, - 114,74,0,0,0,90,23,68,69,66,85,71,95,66,89,84, - 69,67,79,68,69,95,83,85,70,70,73,88,69,83,90,27, - 79,80,84,73,77,73,90,69,68,95,66,89,84,69,67,79, - 68,69,95,83,85,70,70,73,88,69,83,114,79,0,0,0, - 114,85,0,0,0,114,91,0,0,0,114,95,0,0,0,114, - 97,0,0,0,114,116,0,0,0,114,123,0,0,0,114,135, - 0,0,0,114,141,0,0,0,114,144,0,0,0,114,149,0, - 0,0,218,6,111,98,106,101,99,116,114,156,0,0,0,114, - 161,0,0,0,114,162,0,0,0,114,178,0,0,0,114,188, - 0,0,0,114,204,0,0,0,114,212,0,0,0,114,217,0, - 0,0,114,223,0,0,0,114,218,0,0,0,114,224,0,0, - 0,114,241,0,0,0,114,243,0,0,0,114,0,1,0,0, - 114,18,1,0,0,114,155,0,0,0,114,27,1,0,0,114, - 29,1,0,0,114,4,0,0,0,114,4,0,0,0,114,4, - 0,0,0,114,5,0,0,0,218,8,60,109,111,100,117,108, - 101,62,8,0,0,0,115,102,0,0,0,6,17,6,3,12, - 12,12,5,12,5,12,6,12,12,12,10,12,9,12,5,12, - 7,15,22,15,112,22,1,18,2,6,1,6,2,9,2,9, - 2,10,2,21,44,12,33,12,19,12,12,12,12,12,28,12, - 17,21,55,21,12,18,10,12,14,9,3,12,1,15,65,19, - 64,19,29,22,110,19,41,25,45,25,16,6,3,25,53,19, - 57,19,42,19,127,0,7,19,127,0,20,15,23,12,11,12, - 68, + 0,0,114,1,1,0,0,184,4,0,0,115,6,0,0,0, + 0,1,15,1,18,1,122,20,70,105,108,101,70,105,110,100, + 101,114,46,95,103,101,116,95,115,112,101,99,78,99,3,0, + 0,0,0,0,0,0,14,0,0,0,15,0,0,0,67,0, + 0,0,115,228,1,0,0,100,1,0,125,3,0,124,1,0, + 106,0,0,100,2,0,131,1,0,100,3,0,25,125,4,0, + 121,34,0,116,1,0,124,0,0,106,2,0,112,49,0,116, + 3,0,106,4,0,131,0,0,131,1,0,106,5,0,125,5, + 0,87,110,24,0,4,116,6,0,107,10,0,114,85,0,1, + 1,1,100,10,0,125,5,0,89,110,1,0,88,124,5,0, + 124,0,0,106,7,0,107,3,0,114,120,0,124,0,0,106, + 8,0,131,0,0,1,124,5,0,124,0,0,95,7,0,116, + 9,0,131,0,0,114,153,0,124,0,0,106,10,0,125,6, + 0,124,4,0,106,11,0,131,0,0,125,7,0,110,15,0, + 124,0,0,106,12,0,125,6,0,124,4,0,125,7,0,124, + 7,0,124,6,0,107,6,0,114,45,1,116,13,0,124,0, + 0,106,2,0,124,4,0,131,2,0,125,8,0,120,100,0, + 124,0,0,106,14,0,68,93,77,0,92,2,0,125,9,0, + 125,10,0,100,5,0,124,9,0,23,125,11,0,116,13,0, + 124,8,0,124,11,0,131,2,0,125,12,0,116,15,0,124, + 12,0,131,1,0,114,208,0,124,0,0,106,16,0,124,10, + 0,124,1,0,124,12,0,124,8,0,103,1,0,124,2,0, + 131,5,0,83,113,208,0,87,116,17,0,124,8,0,131,1, + 0,125,3,0,120,120,0,124,0,0,106,14,0,68,93,109, + 0,92,2,0,125,9,0,125,10,0,116,13,0,124,0,0, + 106,2,0,124,4,0,124,9,0,23,131,2,0,125,12,0, + 116,18,0,106,19,0,100,6,0,124,12,0,100,7,0,100, + 3,0,131,2,1,1,124,7,0,124,9,0,23,124,6,0, + 107,6,0,114,55,1,116,15,0,124,12,0,131,1,0,114, + 55,1,124,0,0,106,16,0,124,10,0,124,1,0,124,12, + 0,100,8,0,124,2,0,131,5,0,83,113,55,1,87,124, + 3,0,114,224,1,116,18,0,106,19,0,100,9,0,124,8, + 0,131,2,0,1,116,18,0,106,20,0,124,1,0,100,8, + 0,131,2,0,125,13,0,124,8,0,103,1,0,124,13,0, + 95,21,0,124,13,0,83,100,8,0,83,41,11,122,125,84, + 114,121,32,116,111,32,102,105,110,100,32,97,32,108,111,97, + 100,101,114,32,102,111,114,32,116,104,101,32,115,112,101,99, + 105,102,105,101,100,32,109,111,100,117,108,101,44,32,111,114, + 32,116,104,101,32,110,97,109,101,115,112,97,99,101,10,32, + 32,32,32,32,32,32,32,112,97,99,107,97,103,101,32,112, + 111,114,116,105,111,110,115,46,32,82,101,116,117,114,110,115, + 32,40,108,111,97,100,101,114,44,32,108,105,115,116,45,111, + 102,45,112,111,114,116,105,111,110,115,41,46,70,114,58,0, + 0,0,114,56,0,0,0,114,29,0,0,0,114,179,0,0, + 0,122,9,116,114,121,105,110,103,32,123,125,90,9,118,101, + 114,98,111,115,105,116,121,78,122,25,112,111,115,115,105,98, + 108,101,32,110,97,109,101,115,112,97,99,101,32,102,111,114, + 32,123,125,114,87,0,0,0,41,22,114,32,0,0,0,114, + 39,0,0,0,114,35,0,0,0,114,3,0,0,0,114,45, + 0,0,0,114,213,0,0,0,114,40,0,0,0,114,4,1, + 0,0,218,11,95,102,105,108,108,95,99,97,99,104,101,114, + 6,0,0,0,114,7,1,0,0,114,88,0,0,0,114,6, + 1,0,0,114,28,0,0,0,114,3,1,0,0,114,44,0, + 0,0,114,1,1,0,0,114,46,0,0,0,114,114,0,0, + 0,114,129,0,0,0,114,154,0,0,0,114,150,0,0,0, + 41,14,114,100,0,0,0,114,119,0,0,0,114,174,0,0, + 0,90,12,105,115,95,110,97,109,101,115,112,97,99,101,90, + 11,116,97,105,108,95,109,111,100,117,108,101,114,126,0,0, + 0,90,5,99,97,99,104,101,90,12,99,97,99,104,101,95, + 109,111,100,117,108,101,90,9,98,97,115,101,95,112,97,116, + 104,114,219,0,0,0,114,159,0,0,0,90,13,105,110,105, + 116,95,102,105,108,101,110,97,109,101,90,9,102,117,108,108, + 95,112,97,116,104,114,158,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,175,0,0,0,189,4, + 0,0,115,70,0,0,0,0,3,6,1,19,1,3,1,34, + 1,13,1,11,1,15,1,10,1,9,2,9,1,9,1,15, + 2,9,1,6,2,12,1,18,1,22,1,10,1,15,1,12, + 1,32,4,12,2,22,1,22,1,22,1,16,1,12,1,15, + 1,14,1,6,1,16,1,18,1,12,1,4,1,122,20,70, + 105,108,101,70,105,110,100,101,114,46,102,105,110,100,95,115, + 112,101,99,99,1,0,0,0,0,0,0,0,9,0,0,0, + 13,0,0,0,67,0,0,0,115,11,1,0,0,124,0,0, + 106,0,0,125,1,0,121,31,0,116,1,0,106,2,0,124, + 1,0,112,33,0,116,1,0,106,3,0,131,0,0,131,1, + 0,125,2,0,87,110,33,0,4,116,4,0,116,5,0,116, + 6,0,102,3,0,107,10,0,114,75,0,1,1,1,103,0, + 0,125,2,0,89,110,1,0,88,116,7,0,106,8,0,106, + 9,0,100,1,0,131,1,0,115,112,0,116,10,0,124,2, + 0,131,1,0,124,0,0,95,11,0,110,111,0,116,10,0, + 131,0,0,125,3,0,120,90,0,124,2,0,68,93,82,0, + 125,4,0,124,4,0,106,12,0,100,2,0,131,1,0,92, + 3,0,125,5,0,125,6,0,125,7,0,124,6,0,114,191, + 0,100,3,0,106,13,0,124,5,0,124,7,0,106,14,0, + 131,0,0,131,2,0,125,8,0,110,6,0,124,5,0,125, + 8,0,124,3,0,106,15,0,124,8,0,131,1,0,1,113, + 128,0,87,124,3,0,124,0,0,95,11,0,116,7,0,106, + 8,0,106,9,0,116,16,0,131,1,0,114,7,1,100,4, + 0,100,5,0,132,0,0,124,2,0,68,131,1,0,124,0, + 0,95,17,0,100,6,0,83,41,7,122,68,70,105,108,108, + 32,116,104,101,32,99,97,99,104,101,32,111,102,32,112,111, + 116,101,110,116,105,97,108,32,109,111,100,117,108,101,115,32, + 97,110,100,32,112,97,99,107,97,103,101,115,32,102,111,114, + 32,116,104,105,115,32,100,105,114,101,99,116,111,114,121,46, + 114,0,0,0,0,114,58,0,0,0,122,5,123,125,46,123, + 125,99,1,0,0,0,0,0,0,0,2,0,0,0,3,0, + 0,0,83,0,0,0,115,28,0,0,0,104,0,0,124,0, + 0,93,18,0,125,1,0,124,1,0,106,0,0,131,0,0, + 146,2,0,113,6,0,83,114,4,0,0,0,41,1,114,88, + 0,0,0,41,2,114,22,0,0,0,90,2,102,110,114,4, + 0,0,0,114,4,0,0,0,114,5,0,0,0,250,9,60, + 115,101,116,99,111,109,112,62,8,5,0,0,115,2,0,0, + 0,9,0,122,41,70,105,108,101,70,105,110,100,101,114,46, + 95,102,105,108,108,95,99,97,99,104,101,46,60,108,111,99, + 97,108,115,62,46,60,115,101,116,99,111,109,112,62,78,41, + 18,114,35,0,0,0,114,3,0,0,0,90,7,108,105,115, + 116,100,105,114,114,45,0,0,0,114,252,0,0,0,218,15, + 80,101,114,109,105,115,115,105,111,110,69,114,114,111,114,218, + 18,78,111,116,65,68,105,114,101,99,116,111,114,121,69,114, + 114,111,114,114,7,0,0,0,114,8,0,0,0,114,9,0, + 0,0,114,5,1,0,0,114,6,1,0,0,114,83,0,0, + 0,114,47,0,0,0,114,88,0,0,0,218,3,97,100,100, + 114,10,0,0,0,114,7,1,0,0,41,9,114,100,0,0, + 0,114,35,0,0,0,90,8,99,111,110,116,101,110,116,115, + 90,21,108,111,119,101,114,95,115,117,102,102,105,120,95,99, + 111,110,116,101,110,116,115,114,241,0,0,0,114,98,0,0, + 0,114,231,0,0,0,114,219,0,0,0,90,8,110,101,119, + 95,110,97,109,101,114,4,0,0,0,114,4,0,0,0,114, + 5,0,0,0,114,9,1,0,0,235,4,0,0,115,34,0, + 0,0,0,2,9,1,3,1,31,1,22,3,11,3,18,1, + 18,7,9,1,13,1,24,1,6,1,27,2,6,1,17,1, + 9,1,18,1,122,22,70,105,108,101,70,105,110,100,101,114, + 46,95,102,105,108,108,95,99,97,99,104,101,99,1,0,0, + 0,0,0,0,0,3,0,0,0,3,0,0,0,7,0,0, + 0,115,25,0,0,0,135,0,0,135,1,0,102,2,0,100, + 1,0,100,2,0,134,0,0,125,2,0,124,2,0,83,41, + 3,97,20,1,0,0,65,32,99,108,97,115,115,32,109,101, + 116,104,111,100,32,119,104,105,99,104,32,114,101,116,117,114, + 110,115,32,97,32,99,108,111,115,117,114,101,32,116,111,32, + 117,115,101,32,111,110,32,115,121,115,46,112,97,116,104,95, + 104,111,111,107,10,32,32,32,32,32,32,32,32,119,104,105, + 99,104,32,119,105,108,108,32,114,101,116,117,114,110,32,97, + 110,32,105,110,115,116,97,110,99,101,32,117,115,105,110,103, + 32,116,104,101,32,115,112,101,99,105,102,105,101,100,32,108, + 111,97,100,101,114,115,32,97,110,100,32,116,104,101,32,112, + 97,116,104,10,32,32,32,32,32,32,32,32,99,97,108,108, + 101,100,32,111,110,32,116,104,101,32,99,108,111,115,117,114, + 101,46,10,10,32,32,32,32,32,32,32,32,73,102,32,116, + 104,101,32,112,97,116,104,32,99,97,108,108,101,100,32,111, + 110,32,116,104,101,32,99,108,111,115,117,114,101,32,105,115, + 32,110,111,116,32,97,32,100,105,114,101,99,116,111,114,121, + 44,32,73,109,112,111,114,116,69,114,114,111,114,32,105,115, + 10,32,32,32,32,32,32,32,32,114,97,105,115,101,100,46, + 10,10,32,32,32,32,32,32,32,32,99,1,0,0,0,0, + 0,0,0,1,0,0,0,4,0,0,0,19,0,0,0,115, + 43,0,0,0,116,0,0,124,0,0,131,1,0,115,30,0, + 116,1,0,100,1,0,100,2,0,124,0,0,131,1,1,130, + 1,0,136,0,0,124,0,0,136,1,0,140,1,0,83,41, + 3,122,45,80,97,116,104,32,104,111,111,107,32,102,111,114, + 32,105,109,112,111,114,116,108,105,98,46,109,97,99,104,105, + 110,101,114,121,46,70,105,108,101,70,105,110,100,101,114,46, + 122,30,111,110,108,121,32,100,105,114,101,99,116,111,114,105, + 101,115,32,97,114,101,32,115,117,112,112,111,114,116,101,100, + 114,35,0,0,0,41,2,114,46,0,0,0,114,99,0,0, + 0,41,1,114,35,0,0,0,41,2,114,164,0,0,0,114, + 8,1,0,0,114,4,0,0,0,114,5,0,0,0,218,24, + 112,97,116,104,95,104,111,111,107,95,102,111,114,95,70,105, + 108,101,70,105,110,100,101,114,20,5,0,0,115,6,0,0, + 0,0,2,12,1,18,1,122,54,70,105,108,101,70,105,110, + 100,101,114,46,112,97,116,104,95,104,111,111,107,46,60,108, + 111,99,97,108,115,62,46,112,97,116,104,95,104,111,111,107, + 95,102,111,114,95,70,105,108,101,70,105,110,100,101,114,114, + 4,0,0,0,41,3,114,164,0,0,0,114,8,1,0,0, + 114,14,1,0,0,114,4,0,0,0,41,2,114,164,0,0, + 0,114,8,1,0,0,114,5,0,0,0,218,9,112,97,116, + 104,95,104,111,111,107,10,5,0,0,115,4,0,0,0,0, + 10,21,6,122,20,70,105,108,101,70,105,110,100,101,114,46, + 112,97,116,104,95,104,111,111,107,99,1,0,0,0,0,0, + 0,0,1,0,0,0,2,0,0,0,67,0,0,0,115,16, + 0,0,0,100,1,0,106,0,0,124,0,0,106,1,0,131, + 1,0,83,41,2,78,122,16,70,105,108,101,70,105,110,100, + 101,114,40,123,33,114,125,41,41,2,114,47,0,0,0,114, + 35,0,0,0,41,1,114,100,0,0,0,114,4,0,0,0, + 114,4,0,0,0,114,5,0,0,0,114,240,0,0,0,28, + 5,0,0,115,2,0,0,0,0,1,122,19,70,105,108,101, + 70,105,110,100,101,114,46,95,95,114,101,112,114,95,95,41, + 15,114,105,0,0,0,114,104,0,0,0,114,106,0,0,0, + 114,107,0,0,0,114,179,0,0,0,114,246,0,0,0,114, + 123,0,0,0,114,176,0,0,0,114,117,0,0,0,114,1, + 1,0,0,114,175,0,0,0,114,9,1,0,0,114,177,0, + 0,0,114,15,1,0,0,114,240,0,0,0,114,4,0,0, + 0,114,4,0,0,0,114,4,0,0,0,114,5,0,0,0, + 114,2,1,0,0,143,4,0,0,115,20,0,0,0,12,7, + 6,2,12,14,12,4,6,2,12,12,12,5,15,46,12,31, + 18,18,114,2,1,0,0,99,4,0,0,0,0,0,0,0, + 6,0,0,0,11,0,0,0,67,0,0,0,115,195,0,0, + 0,124,0,0,106,0,0,100,1,0,131,1,0,125,4,0, + 124,0,0,106,0,0,100,2,0,131,1,0,125,5,0,124, + 4,0,115,99,0,124,5,0,114,54,0,124,5,0,106,1, + 0,125,4,0,110,45,0,124,2,0,124,3,0,107,2,0, + 114,84,0,116,2,0,124,1,0,124,2,0,131,2,0,125, + 4,0,110,15,0,116,3,0,124,1,0,124,2,0,131,2, + 0,125,4,0,124,5,0,115,126,0,116,4,0,124,1,0, + 124,2,0,100,3,0,124,4,0,131,2,1,125,5,0,121, + 44,0,124,5,0,124,0,0,100,2,0,60,124,4,0,124, + 0,0,100,1,0,60,124,2,0,124,0,0,100,4,0,60, + 124,3,0,124,0,0,100,5,0,60,87,110,18,0,4,116, + 5,0,107,10,0,114,190,0,1,1,1,89,110,1,0,88, + 100,0,0,83,41,6,78,218,10,95,95,108,111,97,100,101, + 114,95,95,218,8,95,95,115,112,101,99,95,95,114,120,0, + 0,0,90,8,95,95,102,105,108,101,95,95,90,10,95,95, + 99,97,99,104,101,100,95,95,41,6,218,3,103,101,116,114, + 120,0,0,0,114,217,0,0,0,114,212,0,0,0,114,161, + 0,0,0,218,9,69,120,99,101,112,116,105,111,110,41,6, + 90,2,110,115,114,98,0,0,0,90,8,112,97,116,104,110, + 97,109,101,90,9,99,112,97,116,104,110,97,109,101,114,120, + 0,0,0,114,158,0,0,0,114,4,0,0,0,114,4,0, + 0,0,114,5,0,0,0,218,14,95,102,105,120,95,117,112, + 95,109,111,100,117,108,101,34,5,0,0,115,34,0,0,0, + 0,2,15,1,15,1,6,1,6,1,12,1,12,1,18,2, + 15,1,6,1,21,1,3,1,10,1,10,1,10,1,14,1, + 13,2,114,20,1,0,0,99,0,0,0,0,0,0,0,0, + 3,0,0,0,3,0,0,0,67,0,0,0,115,55,0,0, + 0,116,0,0,116,1,0,106,2,0,131,0,0,102,2,0, + 125,0,0,116,3,0,116,4,0,102,2,0,125,1,0,116, + 5,0,116,6,0,102,2,0,125,2,0,124,0,0,124,1, + 0,124,2,0,103,3,0,83,41,1,122,95,82,101,116,117, + 114,110,115,32,97,32,108,105,115,116,32,111,102,32,102,105, + 108,101,45,98,97,115,101,100,32,109,111,100,117,108,101,32, + 108,111,97,100,101,114,115,46,10,10,32,32,32,32,69,97, + 99,104,32,105,116,101,109,32,105,115,32,97,32,116,117,112, + 108,101,32,40,108,111,97,100,101,114,44,32,115,117,102,102, + 105,120,101,115,41,46,10,32,32,32,32,41,7,114,218,0, + 0,0,114,139,0,0,0,218,18,101,120,116,101,110,115,105, + 111,110,95,115,117,102,102,105,120,101,115,114,212,0,0,0, + 114,84,0,0,0,114,217,0,0,0,114,74,0,0,0,41, + 3,90,10,101,120,116,101,110,115,105,111,110,115,90,6,115, + 111,117,114,99,101,90,8,98,121,116,101,99,111,100,101,114, + 4,0,0,0,114,4,0,0,0,114,5,0,0,0,114,155, + 0,0,0,57,5,0,0,115,8,0,0,0,0,5,18,1, + 12,1,12,1,114,155,0,0,0,99,1,0,0,0,0,0, + 0,0,12,0,0,0,12,0,0,0,67,0,0,0,115,70, + 2,0,0,124,0,0,97,0,0,116,0,0,106,1,0,97, + 1,0,116,0,0,106,2,0,97,2,0,116,1,0,106,3, + 0,116,4,0,25,125,1,0,120,76,0,100,26,0,68,93, + 68,0,125,2,0,124,2,0,116,1,0,106,3,0,107,7, + 0,114,83,0,116,0,0,106,5,0,124,2,0,131,1,0, + 125,3,0,110,13,0,116,1,0,106,3,0,124,2,0,25, + 125,3,0,116,6,0,124,1,0,124,2,0,124,3,0,131, + 3,0,1,113,44,0,87,100,5,0,100,6,0,103,1,0, + 102,2,0,100,7,0,100,8,0,100,6,0,103,2,0,102, + 2,0,102,2,0,125,4,0,120,149,0,124,4,0,68,93, + 129,0,92,2,0,125,5,0,125,6,0,116,7,0,100,9, + 0,100,10,0,132,0,0,124,6,0,68,131,1,0,131,1, + 0,115,199,0,116,8,0,130,1,0,124,6,0,100,11,0, + 25,125,7,0,124,5,0,116,1,0,106,3,0,107,6,0, + 114,241,0,116,1,0,106,3,0,124,5,0,25,125,8,0, + 80,113,156,0,121,20,0,116,0,0,106,5,0,124,5,0, + 131,1,0,125,8,0,80,87,113,156,0,4,116,9,0,107, + 10,0,114,28,1,1,1,1,119,156,0,89,113,156,0,88, + 113,156,0,87,116,9,0,100,12,0,131,1,0,130,1,0, + 116,6,0,124,1,0,100,13,0,124,8,0,131,3,0,1, + 116,6,0,124,1,0,100,14,0,124,7,0,131,3,0,1, + 116,6,0,124,1,0,100,15,0,100,16,0,106,10,0,124, + 6,0,131,1,0,131,3,0,1,121,19,0,116,0,0,106, + 5,0,100,17,0,131,1,0,125,9,0,87,110,24,0,4, + 116,9,0,107,10,0,114,147,1,1,1,1,100,18,0,125, + 9,0,89,110,1,0,88,116,6,0,124,1,0,100,17,0, + 124,9,0,131,3,0,1,116,0,0,106,5,0,100,19,0, + 131,1,0,125,10,0,116,6,0,124,1,0,100,19,0,124, + 10,0,131,3,0,1,124,5,0,100,7,0,107,2,0,114, + 238,1,116,0,0,106,5,0,100,20,0,131,1,0,125,11, + 0,116,6,0,124,1,0,100,21,0,124,11,0,131,3,0, + 1,116,6,0,124,1,0,100,22,0,116,11,0,131,0,0, + 131,3,0,1,116,12,0,106,13,0,116,2,0,106,14,0, + 131,0,0,131,1,0,1,124,5,0,100,7,0,107,2,0, + 114,66,2,116,15,0,106,16,0,100,23,0,131,1,0,1, + 100,24,0,116,12,0,107,6,0,114,66,2,100,25,0,116, + 17,0,95,18,0,100,18,0,83,41,27,122,205,83,101,116, + 117,112,32,116,104,101,32,112,97,116,104,45,98,97,115,101, + 100,32,105,109,112,111,114,116,101,114,115,32,102,111,114,32, + 105,109,112,111,114,116,108,105,98,32,98,121,32,105,109,112, + 111,114,116,105,110,103,32,110,101,101,100,101,100,10,32,32, + 32,32,98,117,105,108,116,45,105,110,32,109,111,100,117,108, + 101,115,32,97,110,100,32,105,110,106,101,99,116,105,110,103, + 32,116,104,101,109,32,105,110,116,111,32,116,104,101,32,103, + 108,111,98,97,108,32,110,97,109,101,115,112,97,99,101,46, + 10,10,32,32,32,32,79,116,104,101,114,32,99,111,109,112, + 111,110,101,110,116,115,32,97,114,101,32,101,120,116,114,97, + 99,116,101,100,32,102,114,111,109,32,116,104,101,32,99,111, + 114,101,32,98,111,111,116,115,116,114,97,112,32,109,111,100, + 117,108,101,46,10,10,32,32,32,32,114,49,0,0,0,114, + 60,0,0,0,218,8,98,117,105,108,116,105,110,115,114,136, + 0,0,0,90,5,112,111,115,105,120,250,1,47,218,2,110, + 116,250,1,92,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,115,0,0,0,115,33,0,0,0,124,0, + 0,93,23,0,125,1,0,116,0,0,124,1,0,131,1,0, + 100,0,0,107,2,0,86,1,113,3,0,100,1,0,83,41, + 2,114,29,0,0,0,78,41,1,114,31,0,0,0,41,2, + 114,22,0,0,0,114,77,0,0,0,114,4,0,0,0,114, + 4,0,0,0,114,5,0,0,0,114,221,0,0,0,93,5, + 0,0,115,2,0,0,0,6,0,122,25,95,115,101,116,117, + 112,46,60,108,111,99,97,108,115,62,46,60,103,101,110,101, + 120,112,114,62,114,59,0,0,0,122,30,105,109,112,111,114, + 116,108,105,98,32,114,101,113,117,105,114,101,115,32,112,111, + 115,105,120,32,111,114,32,110,116,114,3,0,0,0,114,25, + 0,0,0,114,21,0,0,0,114,30,0,0,0,90,7,95, + 116,104,114,101,97,100,78,90,8,95,119,101,97,107,114,101, + 102,90,6,119,105,110,114,101,103,114,163,0,0,0,114,6, + 0,0,0,122,4,46,112,121,119,122,6,95,100,46,112,121, + 100,84,41,4,122,3,95,105,111,122,9,95,119,97,114,110, + 105,110,103,115,122,8,98,117,105,108,116,105,110,115,122,7, + 109,97,114,115,104,97,108,41,19,114,114,0,0,0,114,7, + 0,0,0,114,139,0,0,0,114,233,0,0,0,114,105,0, + 0,0,90,18,95,98,117,105,108,116,105,110,95,102,114,111, + 109,95,110,97,109,101,114,109,0,0,0,218,3,97,108,108, + 218,14,65,115,115,101,114,116,105,111,110,69,114,114,111,114, + 114,99,0,0,0,114,26,0,0,0,114,11,0,0,0,114, + 223,0,0,0,114,143,0,0,0,114,21,1,0,0,114,84, + 0,0,0,114,157,0,0,0,114,162,0,0,0,114,167,0, + 0,0,41,12,218,17,95,98,111,111,116,115,116,114,97,112, + 95,109,111,100,117,108,101,90,11,115,101,108,102,95,109,111, + 100,117,108,101,90,12,98,117,105,108,116,105,110,95,110,97, + 109,101,90,14,98,117,105,108,116,105,110,95,109,111,100,117, + 108,101,90,10,111,115,95,100,101,116,97,105,108,115,90,10, + 98,117,105,108,116,105,110,95,111,115,114,21,0,0,0,114, + 25,0,0,0,90,9,111,115,95,109,111,100,117,108,101,90, + 13,116,104,114,101,97,100,95,109,111,100,117,108,101,90,14, + 119,101,97,107,114,101,102,95,109,111,100,117,108,101,90,13, + 119,105,110,114,101,103,95,109,111,100,117,108,101,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,6,95,115, + 101,116,117,112,68,5,0,0,115,82,0,0,0,0,8,6, + 1,9,1,9,3,13,1,13,1,15,1,18,2,13,1,20, + 3,33,1,19,2,31,1,10,1,15,1,13,1,4,2,3, + 1,15,1,5,1,13,1,12,2,12,1,16,1,16,1,25, + 3,3,1,19,1,13,2,11,1,16,3,15,1,16,3,12, + 1,15,1,16,3,19,1,19,1,12,1,13,1,12,1,114, + 29,1,0,0,99,1,0,0,0,0,0,0,0,2,0,0, + 0,3,0,0,0,67,0,0,0,115,116,0,0,0,116,0, + 0,124,0,0,131,1,0,1,116,1,0,131,0,0,125,1, + 0,116,2,0,106,3,0,106,4,0,116,5,0,106,6,0, + 124,1,0,140,0,0,103,1,0,131,1,0,1,116,7,0, + 106,8,0,100,1,0,107,2,0,114,78,0,116,2,0,106, + 9,0,106,10,0,116,11,0,131,1,0,1,116,2,0,106, + 9,0,106,10,0,116,12,0,131,1,0,1,116,5,0,124, + 0,0,95,5,0,116,13,0,124,0,0,95,13,0,100,2, + 0,83,41,3,122,41,73,110,115,116,97,108,108,32,116,104, + 101,32,112,97,116,104,45,98,97,115,101,100,32,105,109,112, + 111,114,116,32,99,111,109,112,111,110,101,110,116,115,46,114, + 24,1,0,0,78,41,14,114,29,1,0,0,114,155,0,0, + 0,114,7,0,0,0,114,250,0,0,0,114,143,0,0,0, + 114,2,1,0,0,114,15,1,0,0,114,3,0,0,0,114, + 105,0,0,0,218,9,109,101,116,97,95,112,97,116,104,114, + 157,0,0,0,114,162,0,0,0,114,245,0,0,0,114,212, + 0,0,0,41,2,114,28,1,0,0,90,17,115,117,112,112, + 111,114,116,101,100,95,108,111,97,100,101,114,115,114,4,0, + 0,0,114,4,0,0,0,114,5,0,0,0,218,8,95,105, + 110,115,116,97,108,108,136,5,0,0,115,16,0,0,0,0, + 2,10,1,9,1,28,1,15,1,16,1,16,4,9,1,114, + 31,1,0,0,41,3,122,3,119,105,110,114,1,0,0,0, + 114,2,0,0,0,41,56,114,107,0,0,0,114,10,0,0, + 0,114,11,0,0,0,114,17,0,0,0,114,19,0,0,0, + 114,28,0,0,0,114,38,0,0,0,114,39,0,0,0,114, + 43,0,0,0,114,44,0,0,0,114,46,0,0,0,114,55, + 0,0,0,218,4,116,121,112,101,218,8,95,95,99,111,100, + 101,95,95,114,138,0,0,0,114,15,0,0,0,114,128,0, + 0,0,114,14,0,0,0,114,18,0,0,0,90,17,95,82, + 65,87,95,77,65,71,73,67,95,78,85,77,66,69,82,114, + 73,0,0,0,114,72,0,0,0,114,84,0,0,0,114,74, + 0,0,0,90,23,68,69,66,85,71,95,66,89,84,69,67, + 79,68,69,95,83,85,70,70,73,88,69,83,90,27,79,80, + 84,73,77,73,90,69,68,95,66,89,84,69,67,79,68,69, + 95,83,85,70,70,73,88,69,83,114,79,0,0,0,114,85, + 0,0,0,114,91,0,0,0,114,95,0,0,0,114,97,0, + 0,0,114,116,0,0,0,114,123,0,0,0,114,135,0,0, + 0,114,141,0,0,0,114,144,0,0,0,114,149,0,0,0, + 218,6,111,98,106,101,99,116,114,156,0,0,0,114,161,0, + 0,0,114,162,0,0,0,114,178,0,0,0,114,188,0,0, + 0,114,204,0,0,0,114,212,0,0,0,114,217,0,0,0, + 114,223,0,0,0,114,218,0,0,0,114,224,0,0,0,114, + 243,0,0,0,114,245,0,0,0,114,2,1,0,0,114,20, + 1,0,0,114,155,0,0,0,114,29,1,0,0,114,31,1, + 0,0,114,4,0,0,0,114,4,0,0,0,114,4,0,0, + 0,114,5,0,0,0,218,8,60,109,111,100,117,108,101,62, + 8,0,0,0,115,102,0,0,0,6,17,6,3,12,12,12, + 5,12,5,12,6,12,12,12,10,12,9,12,5,12,7,15, + 22,15,112,22,1,18,2,6,1,6,2,9,2,9,2,10, + 2,21,44,12,33,12,19,12,12,12,12,12,28,12,17,21, + 55,21,12,18,10,12,14,9,3,12,1,15,65,19,64,19, + 29,22,110,19,41,25,45,25,16,6,3,25,53,19,60,19, + 42,19,127,0,7,19,127,0,20,15,23,12,11,12,68, }; -- cgit v0.12 From 32f2eb4941db115d4d5d0902ba086820406ef4b2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 17 Mar 2016 07:50:22 +0000 Subject: Issue #21042: Revert Linux find_library() to return just filename This reverts most of revision 3092cf163eb4. The change worked on x86 architectures, but did not work on ARM, probably due to extra ABI flags in the ldconfig output. --- Doc/library/ctypes.rst | 11 ++++------- Lib/ctypes/test/test_find.py | 40 ++++++++++++++++------------------------ Lib/ctypes/util.py | 2 +- Misc/NEWS | 3 --- 4 files changed, 21 insertions(+), 35 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 4a7309e..828d7ca4 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1258,15 +1258,15 @@ The exact functionality is system dependent. On Linux, :func:`find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, and ``objdump``) to find the library file. It -returns the absolute path of the library file. Here are some examples:: +returns the filename of the library file. Here are some examples:: >>> from ctypes.util import find_library >>> find_library("m") - '/lib/x86_64-linux-gnu/libm.so.6' + 'libm.so.6' >>> find_library("c") - '/lib/x86_64-linux-gnu/libc.so.6' + 'libc.so.6' >>> find_library("bz2") - '/lib/x86_64-linux-gnu/libbz2.so.1.0' + 'libbz2.so.1.0' >>> On OS X, :func:`find_library` tries several predefined naming schemes and paths @@ -1835,9 +1835,6 @@ Utility functions The exact functionality is system dependent. - .. versionchanged:: 3.6 - On Linux it returns an absolute path. - .. function:: find_msvcrt() :module: ctypes.util diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index 1845bb0..e6bc19d 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -9,39 +9,39 @@ from ctypes.util import find_library class Test_OpenGL_libs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.lib_gl = cls.lib_glu = cls.lib_gle = None + lib_gl = lib_glu = lib_gle = None if sys.platform == "win32": - cls.lib_gl = find_library("OpenGL32") - cls.lib_glu = find_library("Glu32") + lib_gl = find_library("OpenGL32") + lib_glu = find_library("Glu32") elif sys.platform == "darwin": - cls.lib_gl = cls.lib_glu = find_library("OpenGL") + lib_gl = lib_glu = find_library("OpenGL") else: - cls.lib_gl = find_library("GL") - cls.lib_glu = find_library("GLU") - cls.lib_gle = find_library("gle") + lib_gl = find_library("GL") + lib_glu = find_library("GLU") + lib_gle = find_library("gle") ## print, for debugging if test.support.verbose: print("OpenGL libraries:") - for item in (("GL", cls.lib_gl), - ("GLU", cls.lib_glu), - ("gle", cls.lib_gle)): + for item in (("GL", lib_gl), + ("GLU", lib_glu), + ("gle", lib_gle)): print("\t", item) cls.gl = cls.glu = cls.gle = None - if cls.lib_gl: + if lib_gl: try: - cls.gl = CDLL(cls.lib_gl, mode=RTLD_GLOBAL) + cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) except OSError: pass - if cls.lib_glu: + if lib_glu: try: - cls.glu = CDLL(cls.lib_glu, RTLD_GLOBAL) + cls.glu = CDLL(lib_glu, RTLD_GLOBAL) except OSError: pass - if cls.lib_gle: + if lib_gle: try: - cls.gle = CDLL(cls.lib_gle) + cls.gle = CDLL(lib_gle) except OSError: pass @@ -64,14 +64,6 @@ class Test_OpenGL_libs(unittest.TestCase): self.skipTest('lib_gle not available') self.gle.gleGetJoinStyle - def test_abspath(self): - if self.lib_gl: - self.assertTrue(os.path.isabs(self.lib_gl)) - if self.lib_glu: - self.assertTrue(os.path.isabs(self.lib_glu)) - if self.lib_gle: - self.assertTrue(os.path.isabs(self.lib_gle)) - # On platforms where the default shared library suffix is '.so', # at least some libraries can be loaded as attributes of the cdll # object, since ctypes now tries loading the lib again diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index d8e3bfa..38bd57f 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -221,7 +221,7 @@ elif os.name == "posix": abi_type = mach_map.get(machine, 'libc6') # XXX assuming GLIBC's ldconfig (with option -p) - regex = r'lib%s\.[^\s]+\s\(%s(?:,\s.*)?\)\s=>\s(.*)' + regex = r'\s+(lib%s\.[^\s]+)\s+\(%s' regex = os.fsencode(regex % (re.escape(name), abi_type)) try: with subprocess.Popen(['/sbin/ldconfig', '-p'], diff --git a/Misc/NEWS b/Misc/NEWS index 8338a84..7f93b2b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -245,9 +245,6 @@ Library - Issue #26177: Fixed the keys() method for Canvas and Scrollbar widgets. -- Issue #21042: Make ctypes.util.find_library() return the full path on - Linux, similar to other platforms. Patch by Tamás Bence Gedai. - - Issue #15068: Got rid of excessive buffering in fileinput. The bufsize parameter is now deprecated and ignored. -- cgit v0.12 From 013024ef67b7e5989e4be03f4ff2be22aa753ae0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 09:43:14 +0100 Subject: Fix compilation error of traceback.c on Windows Issue #26564. --- Python/traceback.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/traceback.c b/Python/traceback.c index a40dbd1..8383c16 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -509,13 +509,13 @@ _Py_DumpDecimal(int fd, unsigned long value) static void dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) { - Py_ssize_t size = sizeof(unsigned long) * 2; - char buffer[size + 1], *ptr, *end; + char buffer[sizeof(unsigned long) * 2 + 1], *ptr, *end; + const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1; if (width > size) width = size; - end = &buffer[Py_ARRAY_LENGTH(buffer) - 1]; + end = &buffer[size]; ptr = end; *ptr = '\0'; do { -- cgit v0.12 From ad524375af042a549d28ec252f3071a595b892b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 12:12:53 +0100 Subject: Fail if PyMem_Malloc() is called without holding the GIL Issue #26563: Debug hooks on Python memory allocators now raise a fatal error if functions of the PyMem_Malloc() family are called without holding the GIL. --- Lib/test/test_capi.py | 17 +++++++++++++---- Misc/NEWS | 4 ++++ Modules/_testcapimodule.c | 19 +++++++++++++++++++ Objects/obmalloc.c | 14 +++++++------- 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 8e6245b..8f4836a 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -602,15 +602,24 @@ class PyMemDebugTests(unittest.TestCase): regex = regex.format(ptr=self.PTR_REGEX) self.assertRegex(out, regex) - def test_pyobject_malloc_without_gil(self): - # Calling PyObject_Malloc() without holding the GIL must raise an - # error in debug mode. - code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + def check_malloc_without_gil(self, code): out = self.check(code) expected = ('Fatal Python error: Python memory allocator called ' 'without holding the GIL') self.assertIn(expected, out) + def test_pymem_malloc_without_gil(self): + # Debug hooks must raise an error if PyMem_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' + self.check_malloc_without_gil(code) + + def test_pyobject_malloc_without_gil(self): + # Debug hooks must raise an error if PyObject_Malloc() is called + # without holding the GIL + code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' + self.check_malloc_without_gil(code) + class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' diff --git a/Misc/NEWS b/Misc/NEWS index 7f93b2b..adfa04b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26563: Debug hooks on Python memory allocators now raise a fatal + error if functions of the :c:func:`PyMem_Malloc` family are called without + holding the GIL. + - Issue #26564: On error, the debug hooks on Python memory allocators now use the :mod:`tracemalloc` module to get the traceback where a memory block was allocated. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b3d8818..0fc7cbc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3644,10 +3644,28 @@ pymem_api_misuse(PyObject *self, PyObject *args) } static PyObject* +pymem_malloc_without_gil(PyObject *self, PyObject *args) +{ + char *buffer; + + /* Deliberate bug to test debug hooks on Python memory allocators: + call PyMem_Malloc() without holding the GIL */ + Py_BEGIN_ALLOW_THREADS + buffer = PyMem_Malloc(10); + Py_END_ALLOW_THREADS + + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + +static PyObject* pyobject_malloc_without_gil(PyObject *self, PyObject *args) { char *buffer; + /* Deliberate bug to test debug hooks on Python memory allocators: + call PyObject_Malloc() without holding the GIL */ Py_BEGIN_ALLOW_THREADS buffer = PyObject_Malloc(10); Py_END_ALLOW_THREADS @@ -3841,6 +3859,7 @@ static PyMethodDef TestMethods[] = { {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, + {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 8812f59..503fcdf 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -198,7 +198,7 @@ static PyMemAllocatorEx _PyMem_Raw = { static PyMemAllocatorEx _PyMem = { #ifdef Py_DEBUG - &_PyMem_Debug.mem, PYRAWDBG_FUNCS + &_PyMem_Debug.mem, PYDBG_FUNCS #else NULL, PYMEM_FUNCS #endif @@ -321,17 +321,17 @@ PyMem_SetupDebugHooks(void) PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } - if (_PyMem.malloc != _PyMem_DebugRawMalloc) { - alloc.ctx = &_PyMem_Debug.mem; - PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); - } - alloc.malloc = _PyMem_DebugMalloc; alloc.calloc = _PyMem_DebugCalloc; alloc.realloc = _PyMem_DebugRealloc; alloc.free = _PyMem_DebugFree; + if (_PyMem.malloc != _PyMem_DebugMalloc) { + alloc.ctx = &_PyMem_Debug.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + } + if (_PyObject.malloc != _PyMem_DebugMalloc) { alloc.ctx = &_PyMem_Debug.obj; PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); -- cgit v0.12 From c36674a2c52ecb30e180b3bcced2b8c529cf72fb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 14:30:16 +0100 Subject: Fix usage of PyMem_Malloc() in os.stat() Issue #26563: Replace PyMem_Malloc() with PyMem_RawMalloc() in the Windows implementation of os.stat(), since the code is called without holding the GIL. --- Modules/posixmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7e89878..65b20be 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1463,7 +1463,7 @@ get_target_path(HANDLE hdl, wchar_t **target_path) if(!buf_size) return FALSE; - buf = PyMem_New(wchar_t, buf_size+1); + buf = (wchar_t *)PyMem_RawMalloc((buf_size + 1) * sizeof(wchar_t)); if (!buf) { SetLastError(ERROR_OUTOFMEMORY); return FALSE; @@ -1473,12 +1473,12 @@ get_target_path(HANDLE hdl, wchar_t **target_path) buf, buf_size, VOLUME_NAME_DOS); if(!result_length) { - PyMem_Free(buf); + PyMem_RawFree(buf); return FALSE; } if(!CloseHandle(hdl)) { - PyMem_Free(buf); + PyMem_RawFree(buf); return FALSE; } @@ -1563,7 +1563,7 @@ win32_xstat_impl(const char *path, struct _Py_stat_struct *result, return -1; code = win32_xstat_impl_w(target_path, result, FALSE); - PyMem_Free(target_path); + PyMem_RawFree(target_path); return code; } } else @@ -1653,7 +1653,7 @@ win32_xstat_impl_w(const wchar_t *path, struct _Py_stat_struct *result, return -1; code = win32_xstat_impl_w(target_path, result, FALSE); - PyMem_Free(target_path); + PyMem_RawFree(target_path); return code; } } else -- cgit v0.12 From 861d9abfcf6a4a34790e4edc5e440a68534137e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 22:45:24 +0100 Subject: faulthandler now works in non-Python threads Issue #26563: * Add _PyGILState_GetInterpreterStateUnsafe() function: the single PyInterpreterState used by this process' GILState implementation. * Enhance _Py_DumpTracebackThreads() to retrieve the interpreter state from autoInterpreterState in last resort. The function now accepts NULL for interp and current_tstate parameters. * test_faulthandler: fix a ResourceWarning when test is interrupted by CTRL+c --- Include/pystate.h | 16 +++++++--- Include/traceback.h | 21 ++++++++++--- Lib/test/test_faulthandler.py | 36 +++++++++++++++-------- Modules/faulthandler.c | 68 ++++++++++++++++++++++++++++++++++++------- Python/pylifecycle.c | 16 +--------- Python/pystate.c | 6 ++++ Python/traceback.c | 49 +++++++++++++++++++++++++++++-- 7 files changed, 164 insertions(+), 48 deletions(-) diff --git a/Include/pystate.h b/Include/pystate.h index 550c332..9423bc7 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -243,15 +243,23 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); */ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); -/* Helper/diagnostic function - return 1 if the current thread - * currently holds the GIL, 0 otherwise - */ #ifndef Py_LIMITED_API /* Issue #26558: Flag to disable PyGILState_Check(). - If set, PyGILState_Check() always return 1. */ + If set to non-zero, PyGILState_Check() always return 1. */ PyAPI_DATA(int) _PyGILState_check_enabled; +/* Helper/diagnostic function - return 1 if the current thread + currently holds the GIL, 0 otherwise. + + The function returns 1 if _PyGILState_check_enabled is non-zero. */ PyAPI_FUNC(int) PyGILState_Check(void); + +/* Unsafe function to get the single PyInterpreterState used by this process' + GILState implementation. + + Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini() + is called. */ +PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); #endif #endif /* #ifdef WITH_THREAD */ diff --git a/Include/traceback.h b/Include/traceback.h index 7c630e3..76e169a 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -53,19 +53,32 @@ PyAPI_DATA(void) _Py_DumpTraceback( PyThreadState *tstate); /* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. Return NULL on success, or an error message on error. + NULL. + + Return NULL on success, or an error message on error. This function is written for debug purpose only. It calls _Py_DumpTraceback() for each thread, and so has the same limitations. It only write the traceback of the first 100 threads: write "..." if there are more threads. + If current_tstate is NULL, the function tries to get the Python thread state + of the current thread. It is not an error if the function is unable to get + the current Python thread state. + + If interp is NULL, the function tries to get the interpreter state from + the current Python thread state, or from + _PyGILState_GetInterpreterStateUnsafe() in last resort. + + It is better to pass NULL to interp and current_tstate, the function tries + different options to retrieve these informations. + This function is signal safe. */ PyAPI_DATA(const char*) _Py_DumpTracebackThreads( - int fd, PyInterpreterState *interp, - PyThreadState *current_thread); - + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate); #ifndef Py_LIMITED_API diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 12969d5..c3cd657 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -58,8 +58,9 @@ class FaultHandlerTests(unittest.TestCase): pass_fds.append(fd) with support.SuppressCrashReport(): process = script_helper.spawn_python('-c', code, pass_fds=pass_fds) - stdout, stderr = process.communicate() - exitcode = process.wait() + with process: + stdout, stderr = process.communicate() + exitcode = process.wait() output = support.strip_python_stderr(stdout) output = output.decode('ascii', 'backslashreplace') if filename: @@ -73,14 +74,11 @@ class FaultHandlerTests(unittest.TestCase): with open(fd, "rb", closefd=False) as fp: output = fp.read() output = output.decode('ascii', 'backslashreplace') - output = re.sub('Current thread 0x[0-9a-f]+', - 'Current thread XXX', - output) return output.splitlines(), exitcode def check_fatal_error(self, code, line_number, name_regex, filename=None, all_threads=True, other_regex=None, - fd=None): + fd=None, know_current_thread=True): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -88,19 +86,22 @@ class FaultHandlerTests(unittest.TestCase): Raise an error if the output doesn't match the expected format. """ if all_threads: - header = 'Current thread XXX (most recent call first)' + if know_current_thread: + header = 'Current thread 0x[0-9a-f]+' + else: + header = 'Thread 0x[0-9a-f]+' else: - header = 'Stack (most recent call first)' + header = 'Stack' regex = """ ^Fatal Python error: {name} - {header}: + {header} \(most recent call first\): File "", line {lineno} in """ regex = dedent(regex.format( lineno=line_number, name=name_regex, - header=re.escape(header))).strip() + header=header)).strip() if other_regex: regex += '|' + other_regex output, exitcode = self.get_output(code, filename=filename, fd=fd) @@ -129,6 +130,17 @@ class FaultHandlerTests(unittest.TestCase): 3, 'Segmentation fault') + @unittest.skipIf(not HAVE_THREADS, 'need threads') + def test_fatal_error_c_thread(self): + self.check_fatal_error(""" + import faulthandler + faulthandler.enable() + faulthandler._fatal_error_c_thread() + """, + 3, + 'in new thread', + know_current_thread=False) + def test_sigabrt(self): self.check_fatal_error(""" import faulthandler @@ -465,7 +477,7 @@ class FaultHandlerTests(unittest.TestCase): File ".*threading.py", line [0-9]+ in _bootstrap_inner File ".*threading.py", line [0-9]+ in _bootstrap - Current thread XXX \(most recent call first\): + Current thread 0x[0-9a-f]+ \(most recent call first\): File "", line {lineno} in dump File "", line 28 in $ """ @@ -637,7 +649,7 @@ class FaultHandlerTests(unittest.TestCase): trace = '\n'.join(trace) if not unregister: if all_threads: - regex = 'Current thread XXX \(most recent call first\):\n' + regex = 'Current thread 0x[0-9a-f]+ \(most recent call first\):\n' else: regex = 'Stack \(most recent call first\):\n' regex = expected_traceback(14, 32, regex) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 45c9fcb..1c247b7 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -202,8 +202,9 @@ faulthandler_get_fileno(PyObject **file_ptr) static PyThreadState* get_thread_state(void) { - PyThreadState *tstate = PyThreadState_Get(); + PyThreadState *tstate = _PyThreadState_UncheckedGet(); if (tstate == NULL) { + /* just in case but very unlikely... */ PyErr_SetString(PyExc_RuntimeError, "unable to get the current thread state"); return NULL; @@ -234,11 +235,12 @@ faulthandler_dump_traceback(int fd, int all_threads, PyGILState_GetThisThreadState(). */ tstate = PyGILState_GetThisThreadState(); #else - tstate = PyThreadState_Get(); + tstate = _PyThreadState_UncheckedGet(); #endif - if (all_threads) - _Py_DumpTracebackThreads(fd, interp, tstate); + if (all_threads) { + (void)_Py_DumpTracebackThreads(fd, NULL, tstate); + } else { if (tstate != NULL) _Py_DumpTraceback(fd, tstate); @@ -272,7 +274,7 @@ faulthandler_dump_traceback_py(PyObject *self, return NULL; if (all_threads) { - errmsg = _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate); if (errmsg != NULL) { PyErr_SetString(PyExc_RuntimeError, errmsg); return NULL; @@ -469,7 +471,6 @@ faulthandler_thread(void *unused) { PyLockStatus st; const char* errmsg; - PyThreadState *current; int ok; #if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) sigset_t set; @@ -489,12 +490,9 @@ faulthandler_thread(void *unused) /* Timeout => dump traceback */ assert(st == PY_LOCK_FAILURE); - /* get the thread holding the GIL, NULL if no thread hold the GIL */ - current = _PyThreadState_UncheckedGet(); - _Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, current); + errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL); ok = (errmsg == NULL); if (thread.exit) @@ -894,7 +892,7 @@ static PyObject * faulthandler_sigsegv(PyObject *self, PyObject *args) { int release_gil = 0; - if (!PyArg_ParseTuple(args, "|i:_read_null", &release_gil)) + if (!PyArg_ParseTuple(args, "|i:_sigsegv", &release_gil)) return NULL; if (release_gil) { @@ -907,6 +905,49 @@ faulthandler_sigsegv(PyObject *self, PyObject *args) Py_RETURN_NONE; } +#ifdef WITH_THREAD +static void +faulthandler_fatal_error_thread(void *plock) +{ + PyThread_type_lock *lock = (PyThread_type_lock *)plock; + + Py_FatalError("in new thread"); + + /* notify the caller that we are done */ + PyThread_release_lock(lock); +} + +static PyObject * +faulthandler_fatal_error_c_thread(PyObject *self, PyObject *args) +{ + long thread; + PyThread_type_lock lock; + + faulthandler_suppress_crash_report(); + + lock = PyThread_allocate_lock(); + if (lock == NULL) + return PyErr_NoMemory(); + + PyThread_acquire_lock(lock, WAIT_LOCK); + + thread = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); + if (thread == -1) { + PyThread_free_lock(lock); + PyErr_SetString(PyExc_RuntimeError, "unable to start the thread"); + return NULL; + } + + /* wait until the thread completes: it will never occur, since Py_FatalError() + exits the process immedialty. */ + PyThread_acquire_lock(lock, WAIT_LOCK); + PyThread_release_lock(lock); + PyThread_free_lock(lock); + + Py_RETURN_NONE; +} +#endif + static PyObject * faulthandler_sigfpe(PyObject *self, PyObject *args) { @@ -1065,6 +1106,11 @@ static PyMethodDef module_methods[] = { "a SIGSEGV or SIGBUS signal depending on the platform")}, {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, PyDoc_STR("_sigsegv(release_gil=False): raise a SIGSEGV signal")}, +#ifdef WITH_THREAD + {"_fatal_error_c_thread", faulthandler_fatal_error_c_thread, METH_NOARGS, + PyDoc_STR("fatal_error_c_thread(): " + "call Py_FatalError() in a new C thread.")}, +#endif {"_sigabrt", faulthandler_sigabrt, METH_NOARGS, PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4fc6a15..41528cd 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1275,25 +1275,11 @@ initstdio(void) static void _Py_FatalError_DumpTracebacks(int fd) { - PyThreadState *tstate; - -#ifdef WITH_THREAD - /* PyGILState_GetThisThreadState() works even if the GIL was released */ - tstate = PyGILState_GetThisThreadState(); -#else - tstate = PyThreadState_GET(); -#endif - if (tstate == NULL) { - /* _Py_DumpTracebackThreads() requires the thread state to display - * frames */ - return; - } - fputc('\n', stderr); fflush(stderr); /* display the current Python stack */ - _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + _Py_DumpTracebackThreads(fd, NULL, NULL); } /* Print the current exception (if an exception is set) with its traceback, diff --git a/Python/pystate.c b/Python/pystate.c index e8026c5..0503f32 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -714,6 +714,12 @@ _PyGILState_Init(PyInterpreterState *i, PyThreadState *t) _PyGILState_NoteThreadState(t); } +PyInterpreterState * +_PyGILState_GetInterpreterStateUnsafe(void) +{ + return autoInterpreterState; +} + void _PyGILState_Fini(void) { diff --git a/Python/traceback.c b/Python/traceback.c index 8383c16..403dba5 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -707,11 +707,56 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) handlers if signals were received. */ const char* _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_thread) + PyThreadState *current_tstate) { PyThreadState *tstate; unsigned int nthreads; +#ifdef WITH_THREAD + if (current_tstate == NULL) { + /* _Py_DumpTracebackThreads() is called from signal handlers by + faulthandler. + + SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals + and are thus delivered to the thread that caused the fault. Get the + Python thread state of the current thread. + + PyThreadState_Get() doesn't give the state of the thread that caused + the fault if the thread released the GIL, and so this function + cannot be used. Read the thread local storage (TLS) instead: call + PyGILState_GetThisThreadState(). */ + current_tstate = PyGILState_GetThisThreadState(); + } + + if (interp == NULL) { + if (current_tstate == NULL) { + interp = _PyGILState_GetInterpreterStateUnsafe(); + if (interp == NULL) { + /* We need the interpreter state to get Python threads */ + return "unable to get the interpreter state"; + } + } + else { + interp = current_tstate->interp; + } + } +#else + if (current_tstate == NULL) { + /* Call _PyThreadState_UncheckedGet() instead of PyThreadState_Get() + to not fail with a fatal error if the thread state is NULL. */ + current_thread = _PyThreadState_UncheckedGet(); + } + + if (interp == NULL) { + if (current_tstate == NULL) { + /* We need the interpreter state to get Python threads */ + return "unable to get the interpreter state"; + } + interp = current_tstate->interp; + } +#endif + assert(interp != NULL); + /* Get the current interpreter from the current thread */ tstate = PyInterpreterState_ThreadHead(interp); if (tstate == NULL) @@ -729,7 +774,7 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PUTS(fd, "...\n"); break; } - write_thread_id(fd, tstate, tstate == current_thread); + write_thread_id(fd, tstate, tstate == current_tstate); dump_traceback(fd, tstate, 0); tstate = PyThreadState_Next(tstate); nthreads++; -- cgit v0.12 From 2025d7839b1d0c5c3cc83601568f16177b1d85b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 23:19:15 +0100 Subject: Py_FatalError: disable faulthandler earlier Issue #26563: Py_FatalError: disable faulthandler before trying to flush sys.stdout and sys.stderr. --- Python/pylifecycle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 41528cd..502a1e6 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1371,17 +1371,17 @@ Py_FatalError(const char *msg) if (!_Py_FatalError_PrintExc(fd)) _Py_FatalError_DumpTracebacks(fd); + /* The main purpose of faulthandler is to display the traceback. We already + * did our best to display it. So faulthandler can now be disabled. + * (Don't trigger it on abort().) */ + _PyFaulthandler_Fini(); + /* Check if the current Python thread hold the GIL */ if (PyThreadState_GET() != NULL) { /* Flush sys.stdout and sys.stderr */ flush_std_files(); } - /* The main purpose of faulthandler is to display the traceback. We already - * did our best to display it. So faulthandler can now be disabled. - * (Don't trigger it on abort().) */ - _PyFaulthandler_Fini(); - #ifdef MS_WINDOWS len = strlen(msg); -- cgit v0.12 From 13be7db34c698010d2a10e7ebbb1503f3495b20c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 16 Mar 2016 23:25:02 +0100 Subject: Fix usage of PyMem_Malloc() in overlapped.c Issue #26563: Replace PyMem_Malloc() with PyMem_RawFree() since PostToQueueCallback() calls PyMem_RawFree() (previously PyMem_Free()) in a new C thread which doesn't hold the GIL. --- Modules/overlapped.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modules/overlapped.c b/Modules/overlapped.c index ef77c88..8e6d397 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -238,7 +238,7 @@ PostToQueueCallback(PVOID lpParameter, BOOL TimerOrWaitFired) PostQueuedCompletionStatus(p->CompletionPort, TimerOrWaitFired, 0, p->Overlapped); /* ignore possible error! */ - PyMem_Free(p); + PyMem_RawFree(p); } PyDoc_STRVAR( @@ -262,7 +262,10 @@ overlapped_RegisterWaitWithQueue(PyObject *self, PyObject *args) &Milliseconds)) return NULL; - pdata = PyMem_Malloc(sizeof(struct PostCallbackData)); + /* Use PyMem_RawMalloc() rather than PyMem_Malloc(), since + PostToQueueCallback() will call PyMem_Free() from a new C thread + which doesn't hold the GIL. */ + pdata = PyMem_RawMalloc(sizeof(struct PostCallbackData)); if (pdata == NULL) return SetFromWindowsErr(0); @@ -273,7 +276,7 @@ overlapped_RegisterWaitWithQueue(PyObject *self, PyObject *args) pdata, Milliseconds, WT_EXECUTEINWAITTHREAD | WT_EXECUTEONLYONCE)) { - PyMem_Free(pdata); + PyMem_RawFree(pdata); return SetFromWindowsErr(0); } -- cgit v0.12 From c2fc56836f6cb02ce6436d9eb0342dc595738e9d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 11:04:31 +0100 Subject: Enhance documentation on malloc debug hooks Issue #26564, #26516, #26563. --- Doc/c-api/memory.rst | 9 ++++++--- Doc/using/cmdline.rst | 16 ++++++++++------ Doc/whatsnew/3.6.rst | 6 ++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 843ccac..1787292 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -346,8 +346,9 @@ Customize Memory Allocators - Detect write before the start of the buffer (buffer underflow) - Detect write after the end of the buffer (buffer overflow) - Check that the :term:`GIL ` is held when - allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: - :c:func:`PyObject_Malloc`) are called + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called On error, the debug hooks use the :mod:`tracemalloc` module to get the traceback where a memory block was allocated. The traceback is only @@ -361,7 +362,9 @@ Customize Memory Allocators .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. + where a memory block was allocated. The debug hooks now also check + if the GIL is hold when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + :c:data:`PYMEM_DOMAIN_MEM` domains are called. .. _pymalloc: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 684ccb6..4555982 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -638,16 +638,20 @@ conflict. Install debug hooks: * ``debug``: install debug hooks on top of the default memory allocator - * ``malloc_debug``: same than ``malloc`` but also install debug hooks - * ``pymalloc_debug``: same than ``malloc`` but also install debug hooks + * ``malloc_debug``: same as ``malloc`` but also install debug hooks + * ``pymalloc_debug``: same as ``pyalloc`` but also install debug hooks + + When is compiled in release mode, the default is ``pymalloc``. When Python + is compiled in debug mode, the default is ``pymalloc_debug``: debug hooks + are installed. + + If Python is configured without ``pymalloc`` support, ``pymalloc`` and + ``pymalloc_debug`` are not available, the default is ``malloc`` in release + mode and ``malloc_debug`` in debug mode. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. - .. note:: - ``pymalloc`` and ``pymalloc_debug`` are not available if Python is - configured without ``pymalloc`` support. - .. versionadded:: 3.6 diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 443e46a..411332f 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -118,8 +118,10 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: * Detect write before the start of the buffer (buffer underflow) * Detect write after the end of the buffer (buffer overflow) * Check that the :term:`GIL ` is held when allocator - functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: - :c:func:`PyObject_Malloc`) are called + functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and + :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. + +Checking if the GIL is hold is also a new feature of Python 3.6. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. -- cgit v0.12 From 9b46a5730268f6f22e13cd0a99adada2932f062b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 15:10:43 +0100 Subject: Doc: fix typos, patch written by Stefan Behnel --- Doc/c-api/memory.rst | 2 +- Doc/using/cmdline.rst | 8 ++++---- Doc/whatsnew/3.6.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 1787292..c2e2442 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -363,7 +363,7 @@ Customize Memory Allocators This function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback where a memory block was allocated. The debug hooks now also check - if the GIL is hold when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 4555982..7ff9361 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -639,11 +639,11 @@ conflict. * ``debug``: install debug hooks on top of the default memory allocator * ``malloc_debug``: same as ``malloc`` but also install debug hooks - * ``pymalloc_debug``: same as ``pyalloc`` but also install debug hooks + * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks - When is compiled in release mode, the default is ``pymalloc``. When Python - is compiled in debug mode, the default is ``pymalloc_debug``: debug hooks - are installed. + When Python is compiled in release mode, the default is ``pymalloc``. When + compiled in debug mode, the default is ``pymalloc_debug`` and the debug hooks + are used automatically. If Python is configured without ``pymalloc`` support, ``pymalloc`` and ``pymalloc_debug`` are not available, the default is ``malloc`` in release diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 411332f..cc63589 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -121,7 +121,7 @@ compiled in release mode using ``PYTHONMALLOC=debug``. Effects of debug hooks: functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. -Checking if the GIL is hold is also a new feature of Python 3.6. +Checking if the GIL is held is also a new feature of Python 3.6. See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python memory allocators. -- cgit v0.12 From 5936313651119b859420930b4126529cd38d277e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 18 Mar 2016 11:54:22 -0700 Subject: Issue #26252: Add an example on how to register a finder --- Doc/library/importlib.rst | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 2bb586c3..1a1348f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1335,7 +1335,7 @@ import, then you should use :func:`importlib.util.find_spec`. if spec is None: print("can't find the itertools module") else: - # If you chose to perform the actual import. + # If you chose to perform the actual import ... module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # Adding the module to sys.modules is optional. @@ -1359,11 +1359,41 @@ To import a Python source file directly, use the following recipe # by name later. sys.modules[module_name] = module +For deep customizations of import, you typically want to implement an +:term:`importer`. This means managing both the :term:`finder` and :term:`loader` +side of things. For finders there are two flavours to choose from depending on +your needs: a :term:`meta path finder` or a :term:`path entry finder`. The +former is what you would put on :attr:`sys.meta_path` while the latter is what +you create using a :term:`path entry hook` on :attr:`sys.path_hooks` which works +with :attr:`sys.path` entries to potentially create a finder. This example will +show you how to register your own importers so that import will use them (for +creating an importer for yourself, read the documentation for the appropriate +classes defined within this package):: + + import importlib.machinery + import sys + + # For illustrative purposes only. + SpamMetaPathFinder = importlib.machinery.PathFinder + SpamPathEntryFinder = importlib.machinery.FileFinder + loader_details = (importlib.machinery.SourceFileLoader, + importlib.machinery.SOURCE_SUFFIXES) + + # Setting up a meta path finder. + # Make sure to put the finder in the proper location in the list in terms of + # priority. + sys.meta_path.append(SpamMetaPathFinder) + + # Setting up a path entry finder. + # Make sure to put the path hook in the proper location in the list in terms + # of priority. + sys.path_hooks.append(SpamPathEntryFinder.path_hook(loader_details)) + Import itself is implemented in Python code, making it possible to expose most of the import machinery through importlib. The following helps illustrate the various APIs that importlib exposes by providing an approximate implementation of -:func:`importlib.import_module` (Python 3.4 and newer for importlib usage, +:func:`importlib.import_module` (Python 3.4 and newer for the importlib usage, Python 3.6 and newer for other parts of the code). :: -- cgit v0.12 From 1231a4615fd447f0988a72a134a1fc5e7d4e8d69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 00:47:17 +0100 Subject: Add _showwarnmsg() and _formatwarnmsg() to warnings Issue #26568: add new _showwarnmsg() and _formatwarnmsg() functions to the warnings module. The C function warn_explicit() now calls warnings._showwarnmsg() with a warnings.WarningMessage as parameter, instead of calling warnings.showwarning() with multiple parameters. _showwarnmsg() calls warnings.showwarning() if warnings.showwarning() was replaced. Same for _formatwarnmsg(): call warnings.formatwarning() if it was replaced. --- Lib/test/test_warnings/__init__.py | 11 ++++++ Lib/warnings.py | 69 +++++++++++++++++++++++++-------- Python/_warnings.c | 78 ++++++++++++++++++++++++++------------ 3 files changed, 117 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index cea9c57..70eae4c 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -651,6 +651,17 @@ class _WarningsTests(BaseTest, unittest.TestCase): result = stream.getvalue() self.assertIn(text, result) + def test_showwarnmsg_missing(self): + # Test that _showwarnmsg() missing is okay. + text = 'del _showwarnmsg test' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + self.assertIn(text, result) + def test_showwarning_not_callable(self): with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) diff --git a/Lib/warnings.py b/Lib/warnings.py index 1d4fb20..f54726a 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -6,24 +6,63 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] - def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" + msg = WarningMessage(message, category, filename, lineno, file, line) + _showwarnmsg(msg) + +def formatwarning(message, category, filename, lineno, line=None): + """Function to format a warning the standard way.""" + msg = WarningMessage(message, category, filename, lineno, None, line) + return _formatwarnmsg(msg) + +# Keep references to check if the functions were replaced +_showwarning = showwarning +_formatwarning = formatwarning + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + showwarning = globals().get('showwarning', _showwarning) + if showwarning is not _showwarning: + # warnings.showwarning() was replaced + if not callable(showwarning): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + showwarning(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + + file = msg.file if file is None: file = sys.stderr if file is None: - # sys.stderr is None when run with pythonw.exe - warnings get lost + # sys.stderr is None when run with pythonw.exe: + # warnings get lost return + text = _formatwarnmsg(msg) try: - file.write(formatwarning(message, category, filename, lineno, line)) + file.write(text) except OSError: - pass # the file (probably stderr) is invalid - this warning gets lost. + # the file (probably stderr) is invalid - this warning gets lost. + pass -def formatwarning(message, category, filename, lineno, line=None): +def _formatwarnmsg(msg): """Function to format a warning the standard way.""" + formatwarning = globals().get('formatwarning', _formatwarning) + if formatwarning is not _formatwarning: + # warnings.formatwarning() was replaced + return formatwarning(msg.message, msg.category, + msg.filename, msg.lineno, line=msg.line) + import linecache - s = "%s:%s: %s: %s\n" % (filename, lineno, category.__name__, message) - line = linecache.getline(filename, lineno) if line is None else line + s = ("%s:%s: %s: %s\n" + % (msg.filename, msg.lineno, msg.category.__name__, + msg.message)) + if msg.line is None: + line = linecache.getline(msg.filename, msg.lineno) + else: + line = msg.line if line: line = line.strip() s += " %s\n" % line @@ -293,17 +332,13 @@ def warn_explicit(message, category, filename, lineno, raise RuntimeError( "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) - if not callable(showwarning): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") # Print message and context - showwarning(message, category, filename, lineno) + msg = WarningMessage(message, category, filename, lineno) + _showwarnmsg(msg) class WarningMessage(object): - """Holds the result of a single showwarning() call.""" - _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", "line") @@ -366,11 +401,12 @@ class catch_warnings(object): self._module.filters = self._filters[:] self._module._filters_mutated() self._showwarning = self._module.showwarning + self._showwarnmsg = self._module._showwarnmsg if self._record: log = [] - def showwarning(*args, **kwargs): - log.append(WarningMessage(*args, **kwargs)) - self._module.showwarning = showwarning + def showarnmsg(msg): + log.append(msg) + self._module._showwarnmsg = showarnmsg return log else: return None @@ -381,6 +417,7 @@ class catch_warnings(object): self._module.filters = self._filters self._module._filters_mutated() self._module.showwarning = self._showwarning + self._module._showwarnmsg = self._showwarnmsg # filters contains a sequence of filter 5-tuples diff --git a/Python/_warnings.c b/Python/_warnings.c index daa1355..a8c3703 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -359,6 +359,56 @@ error: PyErr_Clear(); } +static int +call_show_warning(PyObject *category, PyObject *text, PyObject *message, + PyObject *filename, int lineno, PyObject *lineno_obj, + PyObject *sourceline) +{ + PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + + show_fn = get_warnings_attr("_showwarnmsg"); + if (show_fn == NULL) { + if (PyErr_Occurred()) + return -1; + show_warning(filename, lineno, text, category, sourceline); + return 0; + } + + if (!PyCallable_Check(show_fn)) { + PyErr_SetString(PyExc_TypeError, + "warnings._showwarnmsg() must be set to a callable"); + goto error; + } + + warnmsg_cls = get_warnings_attr("WarningMessage"); + if (warnmsg_cls == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get warnings.WarningMessage"); + goto error; + } + + msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category, + filename, lineno_obj, + NULL); + Py_DECREF(warnmsg_cls); + if (msg == NULL) + goto error; + + res = PyObject_CallFunctionObjArgs(show_fn, msg, NULL); + Py_DECREF(show_fn); + Py_DECREF(msg); + + if (res == NULL) + return -1; + + Py_DECREF(res); + return 0; + +error: + Py_XDECREF(show_fn); + return -1; +} + static PyObject * warn_explicit(PyObject *category, PyObject *message, PyObject *filename, int lineno, @@ -470,31 +520,9 @@ warn_explicit(PyObject *category, PyObject *message, if (rc == 1) /* Already warned for this module. */ goto return_none; if (rc == 0) { - PyObject *show_fxn = get_warnings_attr("showwarning"); - if (show_fxn == NULL) { - if (PyErr_Occurred()) - goto cleanup; - show_warning(filename, lineno, text, category, sourceline); - } - else { - PyObject *res; - - if (!PyCallable_Check(show_fxn)) { - PyErr_SetString(PyExc_TypeError, - "warnings.showwarning() must be set to a " - "callable"); - Py_DECREF(show_fxn); - goto cleanup; - } - - res = PyObject_CallFunctionObjArgs(show_fxn, message, category, - filename, lineno_obj, - NULL); - Py_DECREF(show_fxn); - Py_XDECREF(res); - if (res == NULL) - goto cleanup; - } + if (call_show_warning(category, text, message, filename, lineno, + lineno_obj, sourceline) < 0) + goto cleanup; } else /* if (rc == -1) */ goto cleanup; -- cgit v0.12 From 914cde89d4c94b0b9206d0fa22322a1142833a56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 01:03:51 +0100 Subject: On ResourceWarning, log traceback where the object was allocated Issue #26567: * Add a new function PyErr_ResourceWarning() function to pass the destroyed object * Add a source attribute to warnings.WarningMessage * Add warnings._showwarnmsg() which uses tracemalloc to get the traceback where source object was allocated. --- Doc/c-api/exceptions.rst | 8 ++++ Doc/library/warnings.rst | 8 +++- Doc/whatsnew/3.6.rst | 34 ++++++++++++++ Include/warnings.h | 7 +++ Lib/test/test_warnings/__init__.py | 30 +++++++++++++ Lib/warnings.py | 22 +++++++-- Misc/NEWS | 5 +++ Modules/_io/fileio.c | 3 +- Modules/posixmodule.c | 4 +- Modules/socketmodule.c | 3 +- Python/_warnings.c | 91 ++++++++++++++++++++++++++------------ 11 files changed, 175 insertions(+), 40 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 1e708a8..57f36ac 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -334,6 +334,14 @@ an error value). .. versionadded:: 3.2 +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + Querying the error indicator ============================ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 8a538ad..4ce88ab 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -319,7 +319,7 @@ Available Functions of the warning message). -.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None) +.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) This is a low-level interface to the functionality of :func:`warn`, passing in explicitly the message, category, filename and line number, and optionally the @@ -335,6 +335,12 @@ Available Functions source for modules found in zipfiles or other non-filesystem import sources). + *source*, if supplied, is the destroyed object which emitted a + :exc:`ResourceWarning`. + + .. versionchanged:: 3.6 + Add the *source* parameter. + .. function:: showwarning(message, category, filename, lineno, file=None, line=None) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index cc63589..b709917 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -258,6 +258,40 @@ urllib.robotparser (Contributed by Nikolay Bogoychev in :issue:`16099`.) +warnings +-------- + +A new optional *source* parameter has been added to the +:func:`warnings.warn_explicit` function: the destroyed object which emitted a +:exc:`ResourceWarning`. A *source* attribute has also been added to +:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:issue:`26568` and :issue:`26567`). + +When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` is now +used to try to retrieve the traceback where the detroyed object was allocated. + +Example with the script ``example.py``:: + + def func(): + f = open(__file__) + f = None + + func() + +Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``:: + + example.py:3: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "example.py", lineno 2 + f = open(__file__) + File "example.py", lineno 5 + func() + +The "Object allocated at" traceback is new and only displayed if +:mod:`tracemalloc` is tracing Python memory allocations. + + zipfile ------- diff --git a/Include/warnings.h b/Include/warnings.h index effb9fad..c1c6992 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -17,6 +17,13 @@ PyAPI_FUNC(int) PyErr_WarnFormat( Py_ssize_t stack_level, const char *format, /* ASCII-encoded string */ ...); + +/* Emit a ResourceWarning warning */ +PyAPI_FUNC(int) PyErr_ResourceWarning( + PyObject *source, + Py_ssize_t stack_level, + const char *format, /* ASCII-encoded string */ + ...); #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyErr_WarnExplicitObject( PyObject *category, diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 70eae4c..a1b3dba 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -2,7 +2,10 @@ from contextlib import contextmanager import linecache import os from io import StringIO +import re import sys +import tempfile +import textwrap import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure @@ -763,12 +766,39 @@ class WarningsDisplayTests(BaseTest): file_object, expected_file_line) self.assertEqual(expect, file_object.getvalue()) + class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = c_warnings class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + def test_tracemalloc(self): + with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile: + tmpfile.write(textwrap.dedent(""" + def func(): + f = open(__file__) + # Emit ResourceWarning + f = None + + func() + """)) + tmpfile.flush() + fname = tmpfile.name + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname) + stderr = res.err.decode('ascii', 'replace') + stderr = re.sub('<.*>', '<...>', stderr) + expected = textwrap.dedent(f''' + {fname}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call first): + File "{fname}", lineno 3 + f = open(__file__) + File "{fname}", lineno 7 + func() + ''').strip() + self.assertEqual(stderr, expected) + class CatchWarningTests(BaseTest): diff --git a/Lib/warnings.py b/Lib/warnings.py index f54726a..1566065 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -2,6 +2,7 @@ import sys + __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings"] @@ -66,6 +67,18 @@ def _formatwarnmsg(msg): if line: line = line.strip() s += " %s\n" % line + if msg.source is not None: + import tracemalloc + tb = tracemalloc.get_object_traceback(msg.source) + if tb is not None: + s += 'Object allocated at (most recent call first):\n' + for frame in tb: + s += (' File "%s", lineno %s\n' + % (frame.filename, frame.lineno)) + line = linecache.getline(frame.filename, frame.lineno) + if line: + line = line.strip() + s += ' %s\n' % line return s def filterwarnings(action, message="", category=Warning, module="", lineno=0, @@ -267,7 +280,8 @@ def warn(message, category=None, stacklevel=1): globals) def warn_explicit(message, category, filename, lineno, - module=None, registry=None, module_globals=None): + module=None, registry=None, module_globals=None, + source=None): lineno = int(lineno) if module is None: module = filename or "" @@ -333,17 +347,17 @@ def warn_explicit(message, category, filename, lineno, "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context - msg = WarningMessage(message, category, filename, lineno) + msg = WarningMessage(message, category, filename, lineno, source) _showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line") + "line", "source") def __init__(self, message, category, filename, lineno, file=None, - line=None): + line=None, source=None): local_values = locals() for attr in self._WARNING_DETAILS: setattr(self, attr, local_values[attr]) diff --git a/Misc/NEWS b/Misc/NEWS index cef2b81..832d1ac 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -226,6 +226,11 @@ Core and Builtins Library ------- +- Issue #26567: Add a new function :c:func:`PyErr_ResourceWarning` function to + pass the destroyed object. Add a *source* attribute to + :class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses + tracemalloc to get the traceback where source object was allocated. + - Issue #26313: ssl.py _load_windows_store_certs fails if windows cert store is empty. Patch by Baji. diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8bf3922..a02a9c1 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -92,8 +92,7 @@ fileio_dealloc_warn(fileio *self, PyObject *source) if (self->fd >= 0 && self->closefd) { PyObject *exc, *val, *tb; PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed file %R", source)) { + if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) self); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65b20be..3f22d14 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12111,8 +12111,8 @@ ScandirIterator_dealloc(ScandirIterator *iterator) */ ++Py_REFCNT(iterator); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed scandir iterator %R", iterator)) { + if (PyErr_ResourceWarning((PyObject *)iterator, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) iterator); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 77a6b31..657b04b 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4170,8 +4170,7 @@ sock_dealloc(PySocketSockObject *s) Py_ssize_t old_refcount = Py_REFCNT(s); ++Py_REFCNT(s); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed %R", s)) + if (PyErr_ResourceWarning(s, 1, "unclosed %R", s)) /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) s); diff --git a/Python/_warnings.c b/Python/_warnings.c index a8c3703..25299fb 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -287,8 +287,8 @@ update_registry(PyObject *registry, PyObject *text, PyObject *category, } static void -show_warning(PyObject *filename, int lineno, PyObject *text, PyObject - *category, PyObject *sourceline) +show_warning(PyObject *filename, int lineno, PyObject *text, + PyObject *category, PyObject *sourceline) { PyObject *f_stderr; PyObject *name; @@ -362,7 +362,7 @@ error: static int call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *filename, int lineno, PyObject *lineno_obj, - PyObject *sourceline) + PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; @@ -388,7 +388,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, } msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category, - filename, lineno_obj, + filename, lineno_obj, Py_None, Py_None, source, NULL); Py_DECREF(warnmsg_cls); if (msg == NULL) @@ -412,7 +412,8 @@ error: static PyObject * warn_explicit(PyObject *category, PyObject *message, PyObject *filename, int lineno, - PyObject *module, PyObject *registry, PyObject *sourceline) + PyObject *module, PyObject *registry, PyObject *sourceline, + PyObject *source) { PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL; PyObject *item = NULL; @@ -521,7 +522,7 @@ warn_explicit(PyObject *category, PyObject *message, goto return_none; if (rc == 0) { if (call_show_warning(category, text, message, filename, lineno, - lineno_obj, sourceline) < 0) + lineno_obj, sourceline, source) < 0) goto cleanup; } else /* if (rc == -1) */ @@ -766,7 +767,8 @@ get_category(PyObject *message, PyObject *category) } static PyObject * -do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) +do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, + PyObject *source) { PyObject *filename, *module, *registry, *res; int lineno; @@ -775,7 +777,7 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level) return NULL; res = warn_explicit(category, message, filename, lineno, module, registry, - NULL); + NULL, source); Py_DECREF(filename); Py_DECREF(registry); Py_DECREF(module); @@ -796,14 +798,15 @@ warnings_warn(PyObject *self, PyObject *args, PyObject *kwds) category = get_category(message, category); if (category == NULL) return NULL; - return do_warn(message, category, stack_level); + return do_warn(message, category, stack_level, NULL); } static PyObject * warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwd_list[] = {"message", "category", "filename", "lineno", - "module", "registry", "module_globals", 0}; + "module", "registry", "module_globals", + "source", 0}; PyObject *message; PyObject *category; PyObject *filename; @@ -811,10 +814,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) PyObject *module = NULL; PyObject *registry = NULL; PyObject *module_globals = NULL; + PyObject *sourceobj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOO:warn_explicit", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOOO:warn_explicit", kwd_list, &message, &category, &filename, &lineno, &module, - ®istry, &module_globals)) + ®istry, &module_globals, &sourceobj)) return NULL; if (module_globals) { @@ -870,14 +874,14 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) /* Handle the warning. */ returned = warn_explicit(category, message, filename, lineno, module, - registry, source_line); + registry, source_line, sourceobj); Py_DECREF(source_list); return returned; } standard_call: return warn_explicit(category, message, filename, lineno, module, - registry, NULL); + registry, NULL, sourceobj); } static PyObject * @@ -892,14 +896,14 @@ warnings_filters_mutated(PyObject *self, PyObject *args) static int warn_unicode(PyObject *category, PyObject *message, - Py_ssize_t stack_level) + Py_ssize_t stack_level, PyObject *source) { PyObject *res; if (category == NULL) category = PyExc_RuntimeWarning; - res = do_warn(message, category, stack_level); + res = do_warn(message, category, stack_level, source); if (res == NULL) return -1; Py_DECREF(res); @@ -907,12 +911,28 @@ warn_unicode(PyObject *category, PyObject *message, return 0; } +static int +_PyErr_WarnFormatV(PyObject *source, + PyObject *category, Py_ssize_t stack_level, + const char *format, va_list vargs) +{ + PyObject *message; + int res; + + message = PyUnicode_FromFormatV(format, vargs); + if (message == NULL) + return -1; + + res = warn_unicode(category, message, stack_level, source); + Py_DECREF(message); + return res; +} + int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) { - int ret; - PyObject *message; + int res; va_list vargs; #ifdef HAVE_STDARG_PROTOTYPES @@ -920,25 +940,38 @@ PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, #else va_start(vargs); #endif - message = PyUnicode_FromFormatV(format, vargs); - if (message != NULL) { - ret = warn_unicode(category, message, stack_level); - Py_DECREF(message); - } - else - ret = -1; + res = _PyErr_WarnFormatV(NULL, category, stack_level, format, vargs); va_end(vargs); - return ret; + return res; } int +PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, + const char *format, ...) +{ + int res; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + res = _PyErr_WarnFormatV(source, PyExc_ResourceWarning, + stack_level, format, vargs); + va_end(vargs); + return res; +} + + +int PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) { int ret; PyObject *message = PyUnicode_FromString(text); if (message == NULL) return -1; - ret = warn_unicode(category, message, stack_level); + ret = warn_unicode(category, message, stack_level, NULL); Py_DECREF(message); return ret; } @@ -964,7 +997,7 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, if (category == NULL) category = PyExc_RuntimeWarning; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); if (res == NULL) return -1; Py_DECREF(res); @@ -1028,7 +1061,7 @@ PyErr_WarnExplicitFormat(PyObject *category, if (message != NULL) { PyObject *res; res = warn_explicit(category, message, filename, lineno, - module, registry, NULL); + module, registry, NULL, NULL); Py_DECREF(message); if (res != NULL) { Py_DECREF(res); -- cgit v0.12 From eedf13fe231b49b10c9a836ae8c9120d215deb84 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:11:56 +0100 Subject: Fix test_logging Issue #26568: Fix implementation of showwarning() and formatwarning() for test_logging. --- Lib/warnings.py | 60 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/Lib/warnings.py b/Lib/warnings.py index 1566065..9fb21a8 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -10,30 +10,14 @@ __all__ = ["warn", "warn_explicit", "showwarning", def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" msg = WarningMessage(message, category, filename, lineno, file, line) - _showwarnmsg(msg) + _showwarnmsg_impl(msg) def formatwarning(message, category, filename, lineno, line=None): """Function to format a warning the standard way.""" msg = WarningMessage(message, category, filename, lineno, None, line) - return _formatwarnmsg(msg) - -# Keep references to check if the functions were replaced -_showwarning = showwarning -_formatwarning = formatwarning - -def _showwarnmsg(msg): - """Hook to write a warning to a file; replace if you like.""" - showwarning = globals().get('showwarning', _showwarning) - if showwarning is not _showwarning: - # warnings.showwarning() was replaced - if not callable(showwarning): - raise TypeError("warnings.showwarning() must be set to a " - "function or method") - - showwarning(msg.message, msg.category, msg.filename, msg.lineno, - msg.file, msg.line) - return + return _formatwarnmsg_impl(msg) +def _showwarnmsg_impl(msg): file = msg.file if file is None: file = sys.stderr @@ -48,14 +32,7 @@ def _showwarnmsg(msg): # the file (probably stderr) is invalid - this warning gets lost. pass -def _formatwarnmsg(msg): - """Function to format a warning the standard way.""" - formatwarning = globals().get('formatwarning', _formatwarning) - if formatwarning is not _formatwarning: - # warnings.formatwarning() was replaced - return formatwarning(msg.message, msg.category, - msg.filename, msg.lineno, line=msg.line) - +def _formatwarnmsg_impl(msg): import linecache s = ("%s:%s: %s: %s\n" % (msg.filename, msg.lineno, msg.category.__name__, @@ -81,6 +58,35 @@ def _formatwarnmsg(msg): s += ' %s\n' % line return s +# Keep a reference to check if the function was replaced +_showwarning = showwarning + +def _showwarnmsg(msg): + """Hook to write a warning to a file; replace if you like.""" + showwarning = globals().get('showwarning', _showwarning) + if showwarning is not _showwarning: + # warnings.showwarning() was replaced + if not callable(showwarning): + raise TypeError("warnings.showwarning() must be set to a " + "function or method") + + showwarning(msg.message, msg.category, msg.filename, msg.lineno, + msg.file, msg.line) + return + _showwarnmsg_impl(msg) + +# Keep a reference to check if the function was replaced +_formatwarning = formatwarning + +def _formatwarnmsg(msg): + """Function to format a warning the standard way.""" + formatwarning = globals().get('formatwarning', _formatwarning) + if formatwarning is not _formatwarning: + # warnings.formatwarning() was replaced + return formatwarning(msg.message, msg.category, + msg.filename, msg.lineno, line=msg.line) + return _formatwarnmsg_impl(msg) + def filterwarnings(action, message="", category=Warning, module="", lineno=0, append=False): """Insert an entry into the list of warnings filters (at the front). -- cgit v0.12 From f664dc58344fc615f8f5649752bf5cb54efe5b3b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:01:48 +0100 Subject: ResourceWarning: Revert change on socket and scandir io.FileIO has a safe implementation of destructor, but not socket nor scandir. --- Modules/posixmodule.c | 4 ++-- Modules/socketmodule.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 3f22d14..65b20be 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12111,8 +12111,8 @@ ScandirIterator_dealloc(ScandirIterator *iterator) */ ++Py_REFCNT(iterator); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_ResourceWarning((PyObject *)iterator, 1, - "unclosed scandir iterator %R", iterator)) { + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) iterator); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 657b04b..77a6b31 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4170,7 +4170,8 @@ sock_dealloc(PySocketSockObject *s) Py_ssize_t old_refcount = Py_REFCNT(s); ++Py_REFCNT(s); PyErr_Fetch(&exc, &val, &tb); - if (PyErr_ResourceWarning(s, 1, "unclosed %R", s)) + if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, + "unclosed %R", s)) /* Spurious errors can appear at shutdown */ if (PyErr_ExceptionMatches(PyExc_Warning)) PyErr_WriteUnraisable((PyObject *) s); -- cgit v0.12 From bfab932971593b6468024e5774653bb2f5047b56 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 02:51:45 +0100 Subject: Try to fix test_warnings on Windows Issue #26567. --- Lib/test/test_warnings/__init__.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index a1b3dba..be80b05 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -4,7 +4,6 @@ import os from io import StringIO import re import sys -import tempfile import textwrap import unittest from test import support @@ -774,8 +773,10 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings def test_tracemalloc(self): - with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile: - tmpfile.write(textwrap.dedent(""" + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, 'w') as fp: + fp.write(textwrap.dedent(""" def func(): f = open(__file__) # Emit ResourceWarning @@ -783,12 +784,12 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): func() """)) - tmpfile.flush() - fname = tmpfile.name - res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname) + + res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', support.TESTFN) + stderr = res.err.decode('ascii', 'replace') stderr = re.sub('<.*>', '<...>', stderr) - expected = textwrap.dedent(f''' + expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> f = None Object allocated at (most recent call first): @@ -796,7 +797,8 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): f = open(__file__) File "{fname}", lineno 7 func() - ''').strip() + ''') + expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) -- cgit v0.12 From 74879e41791c9c033597250476d556815921a8e3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:00:08 +0100 Subject: Try again to fix test_warnings on Windows Issue #26567: normalize newlines in test_tracemalloc. --- Lib/test/test_warnings/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index be80b05..e6f47cd 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -788,6 +788,8 @@ class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', support.TESTFN) stderr = res.err.decode('ascii', 'replace') + # normalize newlines + stderr = '\n'.join(stderr.splitlines()) stderr = re.sub('<.*>', '<...>', stderr) expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> -- cgit v0.12 From ee803a8d2ccf50e60d18ca1bb46f7a1cf3db66f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:33:25 +0100 Subject: Issue #26567: enhance ResourceWarning example --- Doc/whatsnew/3.6.rst | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b709917..1129cb3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -272,24 +272,27 @@ used to try to retrieve the traceback where the detroyed object was allocated. Example with the script ``example.py``:: + import warnings + def func(): - f = open(__file__) - f = None + return open(__file__) - func() + f = func() + f = None Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``:: - example.py:3: ResourceWarning: unclosed file <...> + example.py:7: ResourceWarning: unclosed file <_io.TextIOWrapper name='example.py' mode='r' encoding='UTF-8'> f = None Object allocated at (most recent call first): - File "example.py", lineno 2 - f = open(__file__) - File "example.py", lineno 5 - func() + File "example.py", lineno 4 + return open(__file__) + File "example.py", lineno 6 + f = func() The "Object allocated at" traceback is new and only displayed if -:mod:`tracemalloc` is tracing Python memory allocations. +:mod:`tracemalloc` is tracing Python memory allocations and if the +:mod:`warnings` was already imported. zipfile -- cgit v0.12 From af4a1f20bafe5bb38215d8b2cb1470bdf8fe4625 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:33:50 +0100 Subject: fix indentation in Py_DECREF() --- Include/object.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/object.h b/Include/object.h index 0e67a4e..a032005 100644 --- a/Include/object.h +++ b/Include/object.h @@ -785,7 +785,7 @@ PyAPI_FUNC(void) _Py_Dealloc(PyObject *); --(_py_decref_tmp)->ob_refcnt != 0) \ _Py_CHECK_REFCNT(_py_decref_tmp) \ else \ - _Py_Dealloc(_py_decref_tmp); \ + _Py_Dealloc(_py_decref_tmp); \ } while (0) /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear -- cgit v0.12 From 3c3d7f4b99b1f62567592ef57216e07677f17d7f Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 19 Mar 2016 11:44:17 +0200 Subject: Issue #18787: spwd.getspnam() now raises a PermissionError if the user doesn't have privileges. --- Doc/library/spwd.rst | 3 +++ Doc/whatsnew/3.6.rst | 2 ++ Lib/test/test_spwd.py | 10 ++++++++++ Misc/NEWS | 3 +++ Modules/spwdmodule.c | 5 ++++- 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst index 58be78f..53f8c09 100644 --- a/Doc/library/spwd.rst +++ b/Doc/library/spwd.rst @@ -54,6 +54,9 @@ The following functions are defined: Return the shadow password database entry for the given user name. + .. versionchanged:: 3.6 + Raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user + doesn't have privileges. .. function:: getspall() diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 1129cb3..9046058 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -471,6 +471,8 @@ Changes in the Python API the exception will stop a single-threaded server. (Contributed by Martin Panter in :issue:`23430`.) +* :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of + :exc:`KeyError` if the user doesn't have privileges. Changes in the C API -------------------- diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index bea7ab1..fca809e 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -56,5 +56,15 @@ class TestSpwdRoot(unittest.TestCase): self.assertRaises(TypeError, spwd.getspnam, bytes_name) +@unittest.skipUnless(hasattr(os, 'geteuid') and os.geteuid() != 0, + 'non-root user required') +class TestSpwdNonRoot(unittest.TestCase): + + def test_getspnam_exception(self): + with self.assertRaises(PermissionError) as cm: + spwd.getspnam('bin') + self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index fe3071e..ae0a100 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -226,6 +226,9 @@ Core and Builtins Library ------- +- Issue #18787: spwd.getspnam() now raises a PermissionError if the user + doesn't have privileges. + - Issue #26560: Avoid potential ValueError in BaseHandler.start_response. Initial patch by Peter Inglesby. diff --git a/Modules/spwdmodule.c b/Modules/spwdmodule.c index 49324d5..e715d01 100644 --- a/Modules/spwdmodule.c +++ b/Modules/spwdmodule.c @@ -137,7 +137,10 @@ spwd_getspnam_impl(PyModuleDef *module, PyObject *arg) if (PyBytes_AsStringAndSize(bytes, &name, NULL) == -1) goto out; if ((p = getspnam(name)) == NULL) { - PyErr_SetString(PyExc_KeyError, "getspnam(): name not found"); + if (errno != 0) + PyErr_SetFromErrno(PyExc_OSError); + else + PyErr_SetString(PyExc_KeyError, "getspnam(): name not found"); goto out; } retval = mkspent(p); -- cgit v0.12 From 51b846c47a9b1db927939ccfb037a5a0ff6ff99c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Mar 2016 21:52:22 +0100 Subject: _tracemalloc: add domain to trace keys * hashtable.h: key has now a variable size * _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces --- Modules/_tracemalloc.c | 186 ++++++++++++++++++++++++++++++++++++------------- Modules/hashtable.c | 105 ++++++++++++++++------------ Modules/hashtable.h | 89 ++++++++++++++++------- Python/marshal.c | 11 +-- 4 files changed, 270 insertions(+), 121 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5752904..73aa53b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -3,6 +3,7 @@ #include "frameobject.h" #include "pythread.h" #include "osdefs.h" +#include /* For offsetof */ /* Trace memory blocks allocated by PyMem_RawMalloc() */ #define TRACE_RAW_MALLOC @@ -54,6 +55,26 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif +typedef unsigned int domain_t; + +/* FIXME: pack also? */ +typedef struct { + void *ptr; + domain_t domain; +} pointer_t; + +/* Size of pointer_t content, it can be smaller than sizeof(pointer_t) */ +#define POINTER_T_SIZE \ + (offsetof(pointer_t, domain) + sizeof(domain_t)) + +#define POINTER_T_FILL_PADDING(key) \ + do { \ + if (POINTER_T_SIZE != sizeof(pointer_t)) { \ + memset((char *)&(key) + POINTER_T_SIZE, 0, \ + sizeof(pointer_t) - POINTER_T_SIZE); \ + } \ + } while (0) + /* Pack the frame_t structure to reduce the memory footprint on 64-bit architectures: 12 bytes instead of 16. This optimization might produce SIGBUS on architectures not supporting unaligned memory accesses (64-bit @@ -196,23 +217,56 @@ set_reentrant(int reentrant) } #endif +static Py_uhash_t +hashtable_hash_pointer(size_t key_size, const void *pkey) +{ + pointer_t ptr; + Py_uhash_t hash; + + assert(sizeof(ptr) == key_size); + ptr = *(pointer_t *)pkey; + + hash = (Py_uhash_t)_Py_HashPointer(ptr.ptr); + hash ^= ptr.domain; + return hash; +} + +static Py_uhash_t +hashtable_hash_pyobject(size_t key_size, const void *pkey) +{ + PyObject *obj; + + assert(key_size == sizeof(PyObject *)); + obj = *(PyObject **)pkey; + + return PyObject_Hash(obj); +} + static int -hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - if (key != NULL && entry->key != NULL) - return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); + PyObject *key, *entry_key; + + assert(sizeof(key) == key_size); + key = *(PyObject **)pkey; + assert(sizeof(entry_key) == key_size); + entry_key = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + + if (key != NULL && entry_key != NULL) + return (PyUnicode_Compare(key, entry_key) == 0); else - return key == entry->key; + return key == entry_key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t data_size, +hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, 0, + return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -230,20 +284,26 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(const void *key) +hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback = key; + const traceback_t *traceback = *(const traceback_t **)pkey; + assert(key_size == sizeof(const traceback_t *)); return traceback->hash; } static int -hashtable_compare_traceback(const traceback_t *traceback1, +hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) { - const traceback_t *traceback2 = he->key; + traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; + assert(sizeof(traceback1) == key_size); + assert(sizeof(traceback2) == key_size); + traceback1 = *(traceback_t **)pkey; + traceback2 = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(he); + if (traceback1->nframe != traceback2->nframe) return 0; @@ -312,15 +372,16 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - filename = (PyObject *)entry->key; + assert(sizeof(filename) == tracemalloc_filenames->key_size); + filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -403,9 +464,10 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - traceback = (traceback_t *)entry->key; + assert(sizeof(traceback) == tracemalloc_tracebacks->key_size); + traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); } else { traceback_t *copy; @@ -422,7 +484,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -435,8 +497,9 @@ traceback_new(void) } static int -tracemalloc_add_trace(void *ptr, size_t size) +tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) { + pointer_t key; traceback_t *traceback; trace_t trace; int res; @@ -445,10 +508,14 @@ tracemalloc_add_trace(void *ptr, size_t size) if (traceback == NULL) return -1; + key.ptr = ptr; + key.domain = 0; + POINTER_T_FILL_PADDING(key); + trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); if (res == 0) { assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); tracemalloc_traced_memory += size; @@ -460,11 +527,16 @@ tracemalloc_add_trace(void *ptr, size_t size) } static void -tracemalloc_remove_trace(void *ptr) +tracemalloc_remove_trace(void *ptr, domain_t domain) { + pointer_t key; trace_t trace; - if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + key.ptr = ptr; + key.domain = domain; + POINTER_T_FILL_PADDING(key); + + if (_Py_HASHTABLE_POP(tracemalloc_traces, key, trace)) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -486,7 +558,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { + if (tracemalloc_add_trace(ptr, 0, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -510,9 +582,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -530,7 +602,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -555,7 +627,7 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } @@ -610,7 +682,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } return ptr2; @@ -689,7 +761,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + tracemalloc_remove_trace(ptr, 0); TABLES_UNLOCK(); } return ptr2; @@ -714,17 +786,27 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *filename = (PyObject *)entry->key; + PyObject *filename; + + assert(sizeof(filename) == ht->key_size); + filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) +traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - traceback_t *traceback = (traceback_t *)entry->key; + traceback_t *traceback; + + assert(sizeof(traceback) == ht->key_size); + traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); + raw_free(traceback); return 0; } @@ -791,16 +873,16 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(0, - (_Py_hashtable_hash_func)PyObject_Hash, + tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, + hashtable_hash_pyobject, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(0, - (_Py_hashtable_hash_func)hashtable_hash_traceback, - (_Py_hashtable_compare_func)hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, + hashtable_hash_traceback, + hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(trace_t), - _Py_hashtable_hash_ptr, + tracemalloc_traces = hashtable_new(sizeof(pointer_t), sizeof(trace_t), + hashtable_hash_pointer, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL @@ -1065,14 +1147,15 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, + void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1087,9 +1170,11 @@ tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, + _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); Py_DECREF(obj); return 0; } @@ -1120,7 +1205,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1152,7 +1237,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) @@ -1162,16 +1247,21 @@ finally: } static traceback_t* -tracemalloc_get_traceback(const void *ptr) +tracemalloc_get_traceback(const void *ptr, domain_t domain) { + pointer_t key; trace_t trace; int found; if (!tracemalloc_config.tracing) return NULL; + key.ptr = (void *)ptr; + key.domain = domain; + POINTER_T_FILL_PADDING(key); + TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); TABLES_UNLOCK(); if (!found) @@ -1202,7 +1292,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(ptr, 0); if (traceback == NULL) Py_RETURN_NONE; @@ -1229,7 +1319,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(ptr, 0); if (traceback == NULL) return; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7de154b..002c0a9 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -88,21 +88,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, } Py_uhash_t -_Py_hashtable_hash_int(const void *key) +_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) { - return (Py_uhash_t)key; -} + void *key; + + assert(key_size == sizeof(void *)); + key = *(void**)pkey; -Py_uhash_t -_Py_hashtable_hash_ptr(const void *key) -{ return (Py_uhash_t)_Py_HashPointer((void *)key); } int -_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - return entry->key == key; + return (memcmp(pkey, _Py_HASHTABLE_ENTRY_KEY(entry), key_size) == 0); } /* makes sure the real size of the buckets array is a power of 2 */ @@ -119,7 +119,8 @@ round_size(size_t s) } _Py_hashtable_t * -_Py_hashtable_new_full(size_t data_size, size_t init_size, +_Py_hashtable_new_full(size_t key_size, size_t data_size, + size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -144,6 +145,7 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, ht->num_buckets = round_size(init_size); ht->entries = 0; + ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -164,11 +166,12 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, } _Py_hashtable_t * -_Py_hashtable_new(size_t data_size, +_Py_hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(key_size, data_size, + HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } @@ -195,7 +198,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); size += ht->get_data_size_func(data); } } @@ -245,17 +248,21 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) /* Get an entry. Return NULL if the key does not exist. */ _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, + size_t key_size, const void *pkey) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; } @@ -263,18 +270,20 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) } static int -_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; previous = entry; } @@ -298,8 +307,8 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da /* Add a new entry to the hash. The key must not be present in the hash table. Return 0 on success, -1 on memory error. */ int -_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, - void *data, size_t data_size) +_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { Py_uhash_t key_hash; size_t index; @@ -310,11 +319,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); assert(entry == NULL); #endif - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -323,11 +332,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return -1; } - entry->key = (void *)key; entry->key_hash = key_hash; + memcpy(_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -340,13 +349,14 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, /* Get data from an entry. Copy entry data into data and return 1 if the entry exists, return 0 if the entry does not exist. */ int -_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, + size_t data_size, void *data) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); @@ -354,22 +364,23 @@ _Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_ } int -_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key, data, data_size); + return _hashtable_pop_entry(ht, key_size, pkey, data, data_size); } /* Delete an entry. The entry must exist. */ void -_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key, NULL, 0); + int found = _hashtable_pop_entry(ht, key_size, pkey, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key, NULL, 0); + (void)_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } @@ -378,7 +389,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) stops if a non-zero value is returned. */ int _Py_hashtable_foreach(_Py_hashtable_t *ht, - int (*func) (_Py_hashtable_entry_t *entry, void *arg), + _Py_hashtable_foreach_func func, void *arg) { _Py_hashtable_entry_t *entry; @@ -386,7 +397,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(entry, arg); + int res = func(ht, entry, arg); if (res) return res; } @@ -397,6 +408,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { + const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -425,7 +437,8 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - assert(ht->hash_func(entry->key) == entry->key_hash); + + assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -446,7 +459,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -465,7 +478,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -479,13 +492,16 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { + const size_t key_size = src->key_size; + const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, + dst = _Py_hashtable_new_full(key_size, data_size, + src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -496,17 +512,20 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, entry->key, - &new_data, src->data_size); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, &new_data); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(entry); - err = _Py_hashtable_set(dst, entry->key, data, src->data_size); + data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, data); } if (err) { _Py_hashtable_destroy(dst); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index a9f9993..aed3ed0 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -20,26 +20,31 @@ typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; - const void *key; Py_uhash_t key_hash; /* data follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) +#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ + ((void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) +#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); -typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); +typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, + const void *pkey); +typedef int (*_Py_hashtable_compare_func) (size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -56,6 +61,7 @@ typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; + size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -67,15 +73,21 @@ typedef struct { } _Py_hashtable_t; /* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); -PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( + size_t key_size, + const void *pkey); +PyAPI_FUNC(int) _Py_hashtable_compare_direct( + size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( + size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -88,40 +100,65 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, + _Py_hashtable_entry_t *entry, + void *arg); PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, void *arg); + _Py_hashtable_foreach_func func, + void *arg); PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey); + +/* Don't call directly this function, + but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +/* Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + PyAPI_FUNC(void) _Py_hashtable_delete( _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + +#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ + _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) #define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) + _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ + _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) #endif /* Py_LIMITED_API */ diff --git a/Python/marshal.c b/Python/marshal.c index 7a4b9d2..64084f4 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_hashtable_get_entry(p->hashtable, v); + entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); @@ -571,7 +571,8 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -582,9 +583,11 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) { - Py_XDECREF(entry->key); + void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry); + assert(ht->key_size == sizeof(entry_key)); + Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From fac395681fb758401d17974f258b17d285336c57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 10:38:58 +0100 Subject: Optimize bytes.replace(b'', b'.') Issue #26574: Optimize bytes.replace(b'', b'.') and bytearray.replace(b'', b'.'): up to 80% faster. Patch written by Josh Snider. --- Doc/whatsnew/3.6.rst | 3 +++ Misc/ACKS | 1 + Misc/NEWS | 3 +++ Objects/bytearrayobject.c | 28 +++++++++++++++++++--------- Objects/bytesobject.c | 28 +++++++++++++++++++--------- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9046058..986c145 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -339,6 +339,9 @@ Optimizations * Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). +* Optimize ``bytes.replace(b'', b'.')`` and ``bytearray.replace(b'', b'.')``: + up to 80% faster. (Contributed by Josh Snider in :issue:`26574`). + Build and C API Changes ======================= diff --git a/Misc/ACKS b/Misc/ACKS index e67f6d1..52eae69 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1376,6 +1376,7 @@ Mark Smith Roy Smith Ryan Smith-Roberts Rafal Smotrzyk +Josh Snider Eric Snow Dirk Soede Nir Soffer diff --git a/Misc/NEWS b/Misc/NEWS index 6f5c7ab..2fa82f3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ Release date: tba Core and Builtins ----------------- +- Issue #26574: Optimize ``bytes.replace(b'', b'.')`` and + ``bytearray.replace(b'', b'.')``. Patch written by Josh Snider. + - Issue #26581: If coding cookie is specified multiple times on a line in Python source code file, only the first one is taken to account. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 9e8ba39..209a641 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1705,17 +1705,27 @@ replace_interleave(PyByteArrayObject *self, self_s = PyByteArray_AS_STRING(self); result_s = PyByteArray_AS_STRING(result); - /* TODO: special case single character, which doesn't need memcpy */ - - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i=0; i 1) { + /* Lay the first one down (guaranteed this will occur) */ Py_MEMCPY(result_s, to_s, to_len); result_s += to_len; + count -= 1; + + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + } + else { + result_s[0] = to_s[0]; + result_s += to_len; + count -= 1; + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + result_s[0] = to_s[0]; + result_s += to_len; + } } /* Copy the rest of the original string */ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 602dea6..5b9006e 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2464,17 +2464,27 @@ replace_interleave(PyBytesObject *self, self_s = PyBytes_AS_STRING(self); result_s = PyBytes_AS_STRING(result); - /* TODO: special case single character, which doesn't need memcpy */ - - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i=0; i 1) { + /* Lay the first one down (guaranteed this will occur) */ Py_MEMCPY(result_s, to_s, to_len); result_s += to_len; + count -= 1; + + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + } + else { + result_s[0] = to_s[0]; + result_s += to_len; + count -= 1; + for (i = 0; i < count; i++) { + *result_s++ = *self_s++; + result_s[0] = to_s[0]; + result_s += to_len; + } } /* Copy the rest of the original string */ -- cgit v0.12 From 322bc12c3142e7816dd34c6c3085929ab29d3ed8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 14:36:39 +0100 Subject: Ooops, revert changeset ea9efa06c137 Change pushed by mistake, the patch is still under review :-/ """ _tracemalloc: add domain to trace keys * hashtable.h: key has now a variable size * _tracemalloc uses (pointer: void*, domain: unsigned int) as key for traces """ --- Modules/_tracemalloc.c | 186 +++++++++++++------------------------------------ Modules/hashtable.c | 105 ++++++++++++---------------- Modules/hashtable.h | 89 +++++++---------------- Python/marshal.c | 11 ++- 4 files changed, 121 insertions(+), 270 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 73aa53b..5752904 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -3,7 +3,6 @@ #include "frameobject.h" #include "pythread.h" #include "osdefs.h" -#include /* For offsetof */ /* Trace memory blocks allocated by PyMem_RawMalloc() */ #define TRACE_RAW_MALLOC @@ -55,26 +54,6 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif -typedef unsigned int domain_t; - -/* FIXME: pack also? */ -typedef struct { - void *ptr; - domain_t domain; -} pointer_t; - -/* Size of pointer_t content, it can be smaller than sizeof(pointer_t) */ -#define POINTER_T_SIZE \ - (offsetof(pointer_t, domain) + sizeof(domain_t)) - -#define POINTER_T_FILL_PADDING(key) \ - do { \ - if (POINTER_T_SIZE != sizeof(pointer_t)) { \ - memset((char *)&(key) + POINTER_T_SIZE, 0, \ - sizeof(pointer_t) - POINTER_T_SIZE); \ - } \ - } while (0) - /* Pack the frame_t structure to reduce the memory footprint on 64-bit architectures: 12 bytes instead of 16. This optimization might produce SIGBUS on architectures not supporting unaligned memory accesses (64-bit @@ -217,56 +196,23 @@ set_reentrant(int reentrant) } #endif -static Py_uhash_t -hashtable_hash_pointer(size_t key_size, const void *pkey) -{ - pointer_t ptr; - Py_uhash_t hash; - - assert(sizeof(ptr) == key_size); - ptr = *(pointer_t *)pkey; - - hash = (Py_uhash_t)_Py_HashPointer(ptr.ptr); - hash ^= ptr.domain; - return hash; -} - -static Py_uhash_t -hashtable_hash_pyobject(size_t key_size, const void *pkey) -{ - PyObject *obj; - - assert(key_size == sizeof(PyObject *)); - obj = *(PyObject **)pkey; - - return PyObject_Hash(obj); -} - static int -hashtable_compare_unicode(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) { - PyObject *key, *entry_key; - - assert(sizeof(key) == key_size); - key = *(PyObject **)pkey; - assert(sizeof(entry_key) == key_size); - entry_key = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); - - if (key != NULL && entry_key != NULL) - return (PyUnicode_Compare(key, entry_key) == 0); + if (key != NULL && entry->key != NULL) + return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); else - return key == entry_key; + return key == entry->key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t key_size, size_t data_size, +hashtable_new(size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(key_size, data_size, 0, + return _Py_hashtable_new_full(data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -284,26 +230,20 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(size_t key_size, const void *pkey) +hashtable_hash_traceback(const void *key) { - const traceback_t *traceback = *(const traceback_t **)pkey; - assert(key_size == sizeof(const traceback_t *)); + const traceback_t *traceback = key; return traceback->hash; } static int -hashtable_compare_traceback(size_t key_size, const void *pkey, +hashtable_compare_traceback(const traceback_t *traceback1, const _Py_hashtable_entry_t *he) { - traceback_t *traceback1, *traceback2; + const traceback_t *traceback2 = he->key; const frame_t *frame1, *frame2; int i; - assert(sizeof(traceback1) == key_size); - assert(sizeof(traceback2) == key_size); - traceback1 = *(traceback_t **)pkey; - traceback2 = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(he); - if (traceback1->nframe != traceback2->nframe) return 0; @@ -372,16 +312,15 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); + entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); if (entry != NULL) { - assert(sizeof(filename) == tracemalloc_filenames->key_size); - filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); + filename = (PyObject *)entry->key; } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { + if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -464,10 +403,9 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); + entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); if (entry != NULL) { - assert(sizeof(traceback) == tracemalloc_tracebacks->key_size); - traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); + traceback = (traceback_t *)entry->key; } else { traceback_t *copy; @@ -484,7 +422,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { + if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -497,9 +435,8 @@ traceback_new(void) } static int -tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) +tracemalloc_add_trace(void *ptr, size_t size) { - pointer_t key; traceback_t *traceback; trace_t trace; int res; @@ -508,14 +445,10 @@ tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) if (traceback == NULL) return -1; - key.ptr = ptr; - key.domain = 0; - POINTER_T_FILL_PADDING(key); - trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); if (res == 0) { assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); tracemalloc_traced_memory += size; @@ -527,16 +460,11 @@ tracemalloc_add_trace(void *ptr, domain_t domain, size_t size) } static void -tracemalloc_remove_trace(void *ptr, domain_t domain) +tracemalloc_remove_trace(void *ptr) { - pointer_t key; trace_t trace; - key.ptr = ptr; - key.domain = domain; - POINTER_T_FILL_PADDING(key); - - if (_Py_HASHTABLE_POP(tracemalloc_traces, key, trace)) { + if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -558,7 +486,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, 0, nelem * elsize) < 0) { + if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -582,9 +510,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); - if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -602,7 +530,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, 0, new_size) < 0) { + if (tracemalloc_add_trace(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -627,7 +555,7 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } @@ -682,7 +610,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } return ptr2; @@ -761,7 +689,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr, 0); + tracemalloc_remove_trace(ptr); TABLES_UNLOCK(); } return ptr2; @@ -786,27 +714,17 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) { - PyObject *filename; - - assert(sizeof(filename) == ht->key_size); - filename = *(PyObject **)_Py_HASHTABLE_ENTRY_KEY(entry); - + PyObject *filename = (PyObject *)entry->key; Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, - void *user_data) +traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) { - traceback_t *traceback; - - assert(sizeof(traceback) == ht->key_size); - traceback = *(traceback_t **)_Py_HASHTABLE_ENTRY_KEY(entry); - + traceback_t *traceback = (traceback_t *)entry->key; raw_free(traceback); return 0; } @@ -873,16 +791,16 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, - hashtable_hash_pyobject, + tracemalloc_filenames = hashtable_new(0, + (_Py_hashtable_hash_func)PyObject_Hash, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, - hashtable_hash_traceback, - hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(0, + (_Py_hashtable_hash_func)hashtable_hash_traceback, + (_Py_hashtable_compare_func)hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(pointer_t), sizeof(trace_t), - hashtable_hash_pointer, + tracemalloc_traces = hashtable_new(sizeof(trace_t), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL @@ -1147,15 +1065,14 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1170,11 +1087,9 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, - _Py_hashtable_entry_t *entry, - void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); Py_DECREF(obj); return 0; } @@ -1205,7 +1120,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1237,7 +1152,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) @@ -1247,21 +1162,16 @@ finally: } static traceback_t* -tracemalloc_get_traceback(const void *ptr, domain_t domain) +tracemalloc_get_traceback(const void *ptr) { - pointer_t key; trace_t trace; int found; if (!tracemalloc_config.tracing) return NULL; - key.ptr = (void *)ptr; - key.domain = domain; - POINTER_T_FILL_PADDING(key); - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); TABLES_UNLOCK(); if (!found) @@ -1292,7 +1202,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr, 0); + traceback = tracemalloc_get_traceback(ptr); if (traceback == NULL) Py_RETURN_NONE; @@ -1319,7 +1229,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr, 0); + traceback = tracemalloc_get_traceback(ptr); if (traceback == NULL) return; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 002c0a9..7de154b 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -88,21 +88,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, } Py_uhash_t -_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) +_Py_hashtable_hash_int(const void *key) { - void *key; - - assert(key_size == sizeof(void *)); - key = *(void**)pkey; + return (Py_uhash_t)key; +} +Py_uhash_t +_Py_hashtable_hash_ptr(const void *key) +{ return (Py_uhash_t)_Py_HashPointer((void *)key); } int -_Py_hashtable_compare_direct(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) { - return (memcmp(pkey, _Py_HASHTABLE_ENTRY_KEY(entry), key_size) == 0); + return entry->key == key; } /* makes sure the real size of the buckets array is a power of 2 */ @@ -119,8 +119,7 @@ round_size(size_t s) } _Py_hashtable_t * -_Py_hashtable_new_full(size_t key_size, size_t data_size, - size_t init_size, +_Py_hashtable_new_full(size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -145,7 +144,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, ht->num_buckets = round_size(init_size); ht->entries = 0; - ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -166,12 +164,11 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, } _Py_hashtable_t * -_Py_hashtable_new(size_t key_size, size_t data_size, +_Py_hashtable_new(size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(key_size, data_size, - HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } @@ -198,7 +195,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); size += ht->get_data_size_func(data); } } @@ -248,21 +245,17 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) /* Get an entry. Return NULL if the key does not exist. */ _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, - size_t key_size, const void *pkey) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - assert(key_size == ht->key_size); - - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) break; } @@ -270,20 +263,18 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, } static int -_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - void *data, size_t data_size) +_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) break; previous = entry; } @@ -307,8 +298,8 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Add a new entry to the hash. The key must not be present in the hash table. Return 0 on success, -1 on memory error. */ int -_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) +_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; @@ -319,11 +310,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key_size, pkey); + entry = _Py_hashtable_get_entry(ht, key); assert(entry == NULL); #endif - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(key); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -332,11 +323,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, return -1; } + entry->key = (void *)key; entry->key_hash = key_hash; - memcpy(_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -349,14 +340,13 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, /* Get data from an entry. Copy entry data into data and return 1 if the entry exists, return 0 if the entry does not exist. */ int -_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, - size_t data_size, void *data) +_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key_size, pkey); + entry = _Py_hashtable_get_entry(ht, key); if (entry == NULL) return 0; _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); @@ -364,23 +354,22 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, } int -_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) +_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key_size, pkey, data, data_size); + return _hashtable_pop_entry(ht, key, data, data_size); } /* Delete an entry. The entry must exist. */ void -_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) +_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key_size, pkey, NULL, 0); + int found = _hashtable_pop_entry(ht, key, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); + (void)_hashtable_pop_entry(ht, key, NULL, 0); #endif } @@ -389,7 +378,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) stops if a non-zero value is returned. */ int _Py_hashtable_foreach(_Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, + int (*func) (_Py_hashtable_entry_t *entry, void *arg), void *arg) { _Py_hashtable_entry_t *entry; @@ -397,7 +386,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(ht, entry, arg); + int res = func(entry, arg); if (res) return res; } @@ -408,7 +397,6 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { - const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -437,8 +425,7 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - - assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); + assert(ht->hash_func(entry->key) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -459,7 +446,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -478,7 +465,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); ht->alloc.free(entry); entry = entry_next; } @@ -492,16 +479,13 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { - const size_t key_size = src->key_size; - const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(key_size, data_size, - src->num_buckets, + dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -512,20 +496,17 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, &new_data); + err = _Py_hashtable_set(dst, entry->key, + &new_data, src->data_size); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, data); + data = _Py_HASHTABLE_ENTRY_DATA(entry); + err = _Py_hashtable_set(dst, entry->key, data, src->data_size); } if (err) { _Py_hashtable_destroy(dst); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index aed3ed0..a9f9993 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -20,31 +20,26 @@ typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; + const void *key; Py_uhash_t key_hash; /* data follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ - ((void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) +#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) -#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) - -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, - const void *pkey); -typedef int (*_Py_hashtable_compare_func) (size_t key_size, - const void *pkey, - const _Py_hashtable_entry_t *he); +typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); +typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -61,7 +56,6 @@ typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; - size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -73,21 +67,15 @@ typedef struct { } _Py_hashtable_t; /* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( - size_t key_size, - const void *pkey); -PyAPI_FUNC(int) _Py_hashtable_compare_direct( - size_t key_size, - const void *pkey, - const _Py_hashtable_entry_t *entry); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); +PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( - size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( - size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -100,65 +88,40 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, - _Py_hashtable_entry_t *entry, - void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, - void *arg); + _Py_hashtable_foreach_func func, void *arg); PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); - -/* Don't call directly this function, - but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ + const void *key); PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - -/* Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - -/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey, - size_t data_size, - void *data); - + const void *key, + void *data, + size_t data_size); PyAPI_FUNC(void) _Py_hashtable_delete( _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); + const void *key); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) - -#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ - _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) + _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) #define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ - _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) #endif /* Py_LIMITED_API */ diff --git a/Python/marshal.c b/Python/marshal.c index 64084f4..7a4b9d2 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,7 +263,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); + entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); @@ -571,8 +571,7 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(void *), sizeof(int), - _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -583,11 +582,9 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) { - void *entry_key = *(void **)_Py_HASHTABLE_ENTRY_KEY(entry); - assert(ht->key_size == sizeof(entry_key)); - Py_XDECREF(entry_key); + Py_XDECREF(entry->key); return 0; } -- cgit v0.12 From 19a8e844e455a26419f35bd4b57d4a7d19b61b69 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 16:36:48 +0100 Subject: Add socket finalizer Issue #26590: Implement a safe finalizer for the _socket.socket type. It now releases the GIL to close the socket. Use PyErr_ResourceWarning() to raise the ResourceWarning to pass the socket object to the warning logger, to get the traceback where the socket was created (allocated). --- Misc/NEWS | 3 +++ Modules/socketmodule.c | 71 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index 2fa82f3..6dfac97 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,9 @@ Core and Builtins Library ------- +- Issue #26590: Implement a safe finalizer for the _socket.socket type. It now + releases the GIL to close the socket. + - Issue #18787: spwd.getspnam() now raises a PermissionError if the user doesn't have privileges. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 77a6b31..ba3cefd 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2564,12 +2564,14 @@ sock_close(PySocketSockObject *s) { SOCKET_T fd; - /* We do not want to retry upon EINTR: see http://lwn.net/Articles/576478/ - * and http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html - * for more details. - */ - if ((fd = s->sock_fd) != -1) { + fd = s->sock_fd; + if (fd != -1) { s->sock_fd = -1; + + /* We do not want to retry upon EINTR: see + http://lwn.net/Articles/576478/ and + http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html + for more details. */ Py_BEGIN_ALLOW_THREADS (void) SOCKETCLOSE(fd); Py_END_ALLOW_THREADS @@ -4163,22 +4165,45 @@ static PyGetSetDef sock_getsetlist[] = { First close the file description. */ static void -sock_dealloc(PySocketSockObject *s) +sock_finalize(PySocketSockObject *s) { + SOCKET_T fd; + PyObject *error_type, *error_value, *error_traceback; + + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + if (s->sock_fd != -1) { - PyObject *exc, *val, *tb; - Py_ssize_t old_refcount = Py_REFCNT(s); - ++Py_REFCNT(s); - PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed %R", s)) + if (PyErr_ResourceWarning((PyObject *)s, 1, "unclosed %R", s)) { /* Spurious errors can appear at shutdown */ - if (PyErr_ExceptionMatches(PyExc_Warning)) - PyErr_WriteUnraisable((PyObject *) s); - PyErr_Restore(exc, val, tb); - (void) SOCKETCLOSE(s->sock_fd); - Py_REFCNT(s) = old_refcount; + if (PyErr_ExceptionMatches(PyExc_Warning)) { + PyErr_WriteUnraisable((PyObject *)s); + } + } + + /* Only close the socket *after* logging the ResourceWarning warning + to allow the logger to call socket methods like + socket.getsockname(). If the socket is closed before, socket + methods fails with the EBADF error. */ + fd = s->sock_fd; + s->sock_fd = -1; + + /* We do not want to retry upon EINTR: see sock_close() */ + Py_BEGIN_ALLOW_THREADS + (void) SOCKETCLOSE(fd); + Py_END_ALLOW_THREADS } + + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); +} + +static void +sock_dealloc(PySocketSockObject *s) +{ + if (PyObject_CallFinalizerFromDealloc((PyObject *)s) < 0) + return; + Py_TYPE(s)->tp_free((PyObject *)s); } @@ -4395,7 +4420,8 @@ static PyTypeObject sock_type = { PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE + | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ sock_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -4415,6 +4441,15 @@ static PyTypeObject sock_type = { PyType_GenericAlloc, /* tp_alloc */ sock_new, /* tp_new */ PyObject_Del, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + (destructor)sock_finalize, /* tp_finalize */ }; -- cgit v0.12 From e0b75b7e87a385ca8b125e35e7c558626811ca99 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 17:26:04 +0100 Subject: Fix test_ssl.test_refcycle() Issue #26590: support.check_warnings() stores warnins, but ResourceWarning now comes with a reference to the socket object which indirectly keeps the socket alive. --- Lib/test/test_ssl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 9a48483..e0c31a8 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -328,7 +328,7 @@ class BasicSocketTests(unittest.TestCase): wr = weakref.ref(ss) with support.check_warnings(("", ResourceWarning)): del ss - self.assertEqual(wr(), None) + self.assertEqual(wr(), None) def test_wrapped_unconnected(self): # Methods on an unconnected SSLSocket propagate the original -- cgit v0.12 From 928bff0b26adb643a7078575c9075b4b709c1b16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 19 Mar 2016 10:36:36 +0100 Subject: cleanup iobase.c casting iobase_finalize to destructor is not needed --- Modules/_io/iobase.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 025007e..e289a10 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -827,7 +827,7 @@ PyTypeObject PyIOBase_Type = { 0, /* tp_weaklist */ 0, /* tp_del */ 0, /* tp_version_tag */ - (destructor)iobase_finalize, /* tp_finalize */ + iobase_finalize, /* tp_finalize */ }; -- cgit v0.12 From 285cf0a6014af147b82a3446d9e088ad0332720d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 22:00:58 +0100 Subject: hashtable.h now supports keys of any size Issue #26588: hashtable.h now supports keys of any size, not only sizeof(void*). It allows to support key larger than sizeof(void*), but also to use less memory for key smaller than sizeof(void*). --- Modules/_tracemalloc.c | 105 ++++++++++++++++++++----------- Modules/hashtable.c | 147 ++++++++++++++++++++++++++----------------- Modules/hashtable.h | 165 +++++++++++++++++++++++++++++++++++++------------ Python/marshal.c | 15 +++-- 4 files changed, 294 insertions(+), 138 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5752904..6799eb6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -196,23 +196,38 @@ set_reentrant(int reentrant) } #endif +static Py_uhash_t +hashtable_hash_pyobject(size_t key_size, const void *pkey) +{ + PyObject *obj; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, obj); + return PyObject_Hash(obj); +} + static int -hashtable_compare_unicode(const void *key, const _Py_hashtable_entry_t *entry) +hashtable_compare_unicode(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - if (key != NULL && entry->key != NULL) - return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); + PyObject *key, *entry_key; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, key); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, entry_key); + + if (key != NULL && entry_key != NULL) + return (PyUnicode_Compare(key, entry_key) == 0); else - return key == entry->key; + return key == entry_key; } static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; static _Py_hashtable_t * -hashtable_new(size_t data_size, +hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, 0, + return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, NULL, NULL, NULL, &hashtable_alloc); } @@ -230,20 +245,25 @@ raw_free(void *ptr) } static Py_uhash_t -hashtable_hash_traceback(const void *key) +hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback = key; + const traceback_t *traceback; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); return traceback->hash; } static int -hashtable_compare_traceback(const traceback_t *traceback1, +hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) { - const traceback_t *traceback2 = he->key; + traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; + _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, he, traceback2); + if (traceback1->nframe != traceback2->nframe) return 0; @@ -312,15 +332,16 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) } /* intern the filename */ - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - filename = (PyObject *)entry->key; + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames->key_size, entry, + filename); } else { /* tracemalloc_filenames is responsible to keep a reference to the filename */ Py_INCREF(filename); - if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_filenames, filename) < 0) { Py_DECREF(filename); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the filename"); @@ -403,9 +424,10 @@ traceback_new(void) traceback->hash = traceback_hash(traceback); /* intern the traceback */ - entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - traceback = (traceback_t *)entry->key; + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks->key_size, entry, + traceback); } else { traceback_t *copy; @@ -422,7 +444,7 @@ traceback_new(void) } memcpy(copy, traceback, traceback_size); - if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { + if (_Py_HASHTABLE_SET_NODATA(tracemalloc_tracebacks, copy) < 0) { raw_free(copy); #ifdef TRACE_DEBUG tracemalloc_error("failed to intern the traceback: putdata failed"); @@ -464,7 +486,7 @@ tracemalloc_remove_trace(void *ptr) { trace_t trace; - if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + if (_Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace)) { assert(tracemalloc_traced_memory >= trace.size); tracemalloc_traced_memory -= trace.size; } @@ -714,17 +736,23 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) #endif /* TRACE_RAW_MALLOC */ static int -tracemalloc_clear_filename(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *filename = (PyObject *)entry->key; + PyObject *filename; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, filename); Py_DECREF(filename); return 0; } static int -traceback_free_traceback(_Py_hashtable_entry_t *entry, void *user_data) +traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *user_data) { - traceback_t *traceback = (traceback_t *)entry->key; + traceback_t *traceback; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, traceback); raw_free(traceback); return 0; } @@ -791,21 +819,20 @@ tracemalloc_init(void) } #endif - tracemalloc_filenames = hashtable_new(0, - (_Py_hashtable_hash_func)PyObject_Hash, + tracemalloc_filenames = hashtable_new(sizeof(PyObject *), 0, + hashtable_hash_pyobject, hashtable_compare_unicode); - tracemalloc_tracebacks = hashtable_new(0, - (_Py_hashtable_hash_func)hashtable_hash_traceback, - (_Py_hashtable_compare_func)hashtable_compare_traceback); + tracemalloc_tracebacks = hashtable_new(sizeof(traceback_t *), 0, + hashtable_hash_traceback, + hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(trace_t), + tracemalloc_traces = hashtable_new(sizeof(void*), sizeof(trace_t), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL - || tracemalloc_traces == NULL) - { + || tracemalloc_traces == NULL) { PyErr_NoMemory(); return -1; } @@ -840,9 +867,9 @@ tracemalloc_deinit(void) tracemalloc_stop(); /* destroy hash tables */ - _Py_hashtable_destroy(tracemalloc_traces); _Py_hashtable_destroy(tracemalloc_tracebacks); _Py_hashtable_destroy(tracemalloc_filenames); + _Py_hashtable_destroy(tracemalloc_traces); #if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) if (tables_lock != NULL) { @@ -935,8 +962,9 @@ tracemalloc_stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - /* release memory */ tracemalloc_clear_traces(); + + /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; } @@ -1065,14 +1093,15 @@ typedef struct { } get_traces_t; static int -tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entry, + void *user_data) { get_traces_t *get_traces = user_data; trace_t *trace; PyObject *tracemalloc_obj; int res; - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(entry); + trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) @@ -1087,9 +1116,11 @@ tracemalloc_get_traces_fill(_Py_hashtable_entry_t *entry, void *user_data) } static int -tracemalloc_pyobject_decref_cb(_Py_hashtable_entry_t *entry, void *user_data) +tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, + _Py_hashtable_entry_t *entry, + void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); Py_DECREF(obj); return 0; } @@ -1120,7 +1151,7 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1152,7 +1183,7 @@ error: finally: if (get_traces.tracebacks != NULL) { _Py_hashtable_foreach(get_traces.tracebacks, - tracemalloc_pyobject_decref_cb, NULL); + tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } if (get_traces.traces != NULL) diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7de154b..d33f0d7 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -1,5 +1,5 @@ -/* The implementation of the hash table (_Py_hashtable_t) is based on the cfuhash - project: +/* The implementation of the hash table (_Py_hashtable_t) is based on the + cfuhash project: http://sourceforge.net/projects/libcfu/ Copyright of cfuhash: @@ -59,7 +59,7 @@ #define ENTRY_NEXT(ENTRY) \ ((_Py_hashtable_entry_t *)_Py_SLIST_ITEM_NEXT(ENTRY)) #define HASHTABLE_ITEM_SIZE(HT) \ - (sizeof(_Py_hashtable_entry_t) + (HT)->data_size) + (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -70,6 +70,7 @@ _Py_slist_init(_Py_slist_t *list) list->head = NULL; } + static void _Py_slist_prepend(_Py_slist_t *list, _Py_slist_item_t *item) { @@ -77,6 +78,7 @@ _Py_slist_prepend(_Py_slist_t *list, _Py_slist_item_t *item) list->head = item; } + static void _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, _Py_slist_item_t *item) @@ -87,24 +89,26 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, list->head = item->next; } -Py_uhash_t -_Py_hashtable_hash_int(const void *key) -{ - return (Py_uhash_t)key; -} Py_uhash_t -_Py_hashtable_hash_ptr(const void *key) +_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) { + void *key; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, key); return (Py_uhash_t)_Py_HashPointer((void *)key); } + int -_Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry) +_Py_hashtable_compare_direct(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) { - return entry->key == key; + const void *pkey2 = _Py_HASHTABLE_ENTRY_KEY(entry); + return (memcmp(pkey, pkey2, key_size) == 0); } + /* makes sure the real size of the buckets array is a power of 2 */ static size_t round_size(size_t s) @@ -118,8 +122,10 @@ round_size(size_t s) return i; } + _Py_hashtable_t * -_Py_hashtable_new_full(size_t data_size, size_t init_size, +_Py_hashtable_new_full(size_t key_size, size_t data_size, + size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, _Py_hashtable_copy_data_func copy_data_func, @@ -144,6 +150,7 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, ht->num_buckets = round_size(init_size); ht->entries = 0; + ht->key_size = key_size; ht->data_size = data_size; buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); @@ -163,16 +170,19 @@ _Py_hashtable_new_full(size_t data_size, size_t init_size, return ht; } + _Py_hashtable_t * -_Py_hashtable_new(size_t data_size, +_Py_hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { - return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, + return _Py_hashtable_new_full(key_size, data_size, + HASHTABLE_MIN_SIZE, hash_func, compare_func, NULL, NULL, NULL, NULL); } + size_t _Py_hashtable_size(_Py_hashtable_t *ht) { @@ -195,7 +205,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { void *data; - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); size += ht->get_data_size_func(data); } } @@ -203,6 +213,7 @@ _Py_hashtable_size(_Py_hashtable_t *ht) return size; } + #ifdef Py_DEBUG void _Py_hashtable_print_stats(_Py_hashtable_t *ht) @@ -243,38 +254,47 @@ _Py_hashtable_print_stats(_Py_hashtable_t *ht) } #endif -/* Get an entry. Return NULL if the key does not exist. */ + _Py_hashtable_entry_t * -_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_get_entry(_Py_hashtable_t *ht, + size_t key_size, const void *pkey) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; } return entry; } + static int -_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + void *data, size_t data_size) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry, *previous; - key_hash = ht->hash_func(key); + assert(key_size == ht->key_size); + + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + if (entry->key_hash == key_hash + && ht->compare_func(key_size, pkey, entry)) break; previous = entry; } @@ -287,7 +307,7 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -295,26 +315,27 @@ _hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t da return 1; } -/* Add a new entry to the hash. The key must not be present in the hash table. - Return 0 on success, -1 on memory error. */ + int -_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, - void *data, size_t data_size) +_Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { Py_uhash_t key_hash; size_t index; _Py_hashtable_entry_t *entry; + assert(key_size == ht->key_size); + assert(data != NULL || data_size == 0); #ifndef NDEBUG /* Don't write the assertion on a single line because it is interesting to know the duplicated entry if the assertion failed. The entry can be read using a debugger. */ - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); assert(entry == NULL); #endif - key_hash = ht->hash_func(key); + key_hash = ht->hash_func(key_size, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -323,11 +344,11 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return -1; } - entry->key = (void *)key; entry->key_hash = key_hash; + memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(entry), data, data_size); + memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -337,48 +358,48 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, return 0; } -/* Get data from an entry. Copy entry data into data and return 1 if the entry - exists, return 0 if the entry does not exist. */ + int -_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, + size_t data_size, void *data) { _Py_hashtable_entry_t *entry; assert(data != NULL); - entry = _Py_hashtable_get_entry(ht, key); + entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); return 1; } + int -_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +_Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, + size_t data_size, void *data) { assert(data != NULL); assert(ht->free_data_func == NULL); - return _hashtable_pop_entry(ht, key, data, data_size); + return _Py_hashtable_pop_entry(ht, key_size, pkey, data, data_size); } -/* Delete an entry. The entry must exist. */ + void -_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) +_Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { #ifndef NDEBUG - int found = _hashtable_pop_entry(ht, key, NULL, 0); + int found = _Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); assert(found); #else - (void)_hashtable_pop_entry(ht, key, NULL, 0); + (void)_Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } -/* Prototype for a pointer to a function to be called foreach - key/value pair in the hash by hashtable_foreach(). Iteration - stops if a non-zero value is returned. */ + int _Py_hashtable_foreach(_Py_hashtable_t *ht, - int (*func) (_Py_hashtable_entry_t *entry, void *arg), + _Py_hashtable_foreach_func func, void *arg) { _Py_hashtable_entry_t *entry; @@ -386,7 +407,7 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, for (hv = 0; hv < ht->num_buckets; hv++) { for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - int res = func(entry, arg); + int res = func(ht, entry, arg); if (res) return res; } @@ -394,9 +415,11 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, return 0; } + static void hashtable_rehash(_Py_hashtable_t *ht) { + const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -425,7 +448,8 @@ hashtable_rehash(_Py_hashtable_t *ht) for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { size_t entry_index; - assert(ht->hash_func(entry->key) == entry->key_hash); + + assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -436,6 +460,7 @@ hashtable_rehash(_Py_hashtable_t *ht) ht->alloc.free(old_buckets); } + void _Py_hashtable_clear(_Py_hashtable_t *ht) { @@ -446,7 +471,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -455,6 +480,7 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) hashtable_rehash(ht); } + void _Py_hashtable_destroy(_Py_hashtable_t *ht) { @@ -465,7 +491,7 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) while (entry) { _Py_slist_item_t *entry_next = entry->next; if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -475,17 +501,20 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) ht->alloc.free(ht); } -/* Return a copy of the hash table */ + _Py_hashtable_t * _Py_hashtable_copy(_Py_hashtable_t *src) { + const size_t key_size = src->key_size; + const size_t data_size = src->data_size; _Py_hashtable_t *dst; _Py_hashtable_entry_t *entry; size_t bucket; int err; void *data, *new_data; - dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, + dst = _Py_hashtable_new_full(key_size, data_size, + src->num_buckets, src->hash_func, src->compare_func, src->copy_data_func, src->free_data_func, src->get_data_size_func, &src->alloc); @@ -496,17 +525,20 @@ _Py_hashtable_copy(_Py_hashtable_t *src) entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); new_data = src->copy_data_func(data); if (new_data != NULL) - err = _Py_hashtable_set(dst, entry->key, - &new_data, src->data_size); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, &new_data); else err = 1; } else { - data = _Py_HASHTABLE_ENTRY_DATA(entry); - err = _Py_hashtable_set(dst, entry->key, data, src->data_size); + data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, + _Py_HASHTABLE_ENTRY_KEY(entry), + data_size, data); } if (err) { _Py_hashtable_destroy(dst); @@ -516,4 +548,3 @@ _Py_hashtable_copy(_Py_hashtable_t *src) } return dst; } - diff --git a/Modules/hashtable.h b/Modules/hashtable.h index a9f9993..6eb5737 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -1,9 +1,10 @@ #ifndef Py_HASHTABLE_H #define Py_HASHTABLE_H - /* The whole API is private */ #ifndef Py_LIMITED_API +/* Single linked list */ + typedef struct _Py_slist_item_s { struct _Py_slist_item_s *next; } _Py_slist_item_t; @@ -16,30 +17,55 @@ typedef struct { #define _Py_SLIST_HEAD(SLIST) (((_Py_slist_t *)SLIST)->head) + +/* _Py_hashtable: table entry */ + typedef struct { /* used by _Py_hashtable_t.buckets to link entries */ _Py_slist_item_t _Py_slist_item; - const void *key; Py_uhash_t key_hash; - /* data follows */ + /* key (key_size bytes) and then data (data_size bytes) follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_DATA(ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t)) +#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ + ((const void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) + +#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ + ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ + (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) + +/* Get a key value from pkey: use memcpy() rather than a pointer dereference + to avoid memory alignment issues. */ +#define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ + do { \ + assert(sizeof(DST_KEY) == (KEY_SIZE)); \ + memcpy(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ + } while (0) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(ENTRY)) +#define _Py_HASHTABLE_ENTRY_READ_KEY(KEY_SIZE, ENTRY, KEY) \ + do { \ + assert(sizeof(KEY) == (KEY_SIZE)); \ + memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ + } while (0) -#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ +#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA_SIZE, DATA) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ + memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ } while (0) -typedef Py_uhash_t (*_Py_hashtable_hash_func) (const void *key); -typedef int (*_Py_hashtable_compare_func) (const void *key, const _Py_hashtable_entry_t *he); + +/* _Py_hashtable: prototypes */ + +typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, + const void *pkey); +typedef int (*_Py_hashtable_compare_func) (size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *he); typedef void* (*_Py_hashtable_copy_data_func)(void *data); typedef void (*_Py_hashtable_free_data_func)(void *data); typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); @@ -52,10 +78,14 @@ typedef struct { void (*free) (void *ptr); } _Py_hashtable_allocator_t; + +/* _Py_hashtable: table */ + typedef struct { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; + size_t key_size; size_t data_size; _Py_hashtable_hash_func hash_func; @@ -66,16 +96,25 @@ typedef struct { _Py_hashtable_allocator_t alloc; } _Py_hashtable_t; -/* hash and compare functions for integers and pointers */ -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); -PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); -PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, const _Py_hashtable_entry_t *entry); +/* hash a pointer (void*) */ +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( + size_t key_size, + const void *pkey); + +/* comparison using memcmp() */ +PyAPI_FUNC(int) _Py_hashtable_compare_direct( + size_t key_size, + const void *pkey, + const _Py_hashtable_entry_t *entry); PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func); + PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( + size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, @@ -84,45 +123,95 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( _Py_hashtable_free_data_func free_data_func, _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator); + +PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); + +/* Return a copy of the hash table */ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); + PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); -PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); -typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_entry_t *entry, void *arg); +typedef int (*_Py_hashtable_foreach_func) (_Py_hashtable_t *ht, + _Py_hashtable_entry_t *entry, + void *arg); +/* Call func() on each entry of the hashtable. + Iteration stops if func() result is non-zero, in this case it's the result + of the call. Otherwise, the function returns 0. */ PyAPI_FUNC(int) _Py_hashtable_foreach( _Py_hashtable_t *ht, - _Py_hashtable_foreach_func func, void *arg); + _Py_hashtable_foreach_func func, + void *arg); + PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); -PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( - _Py_hashtable_t *ht, - const void *key); +/* Add a new entry to the hash. The key must not be present in the hash table. + Return 0 on success, -1 on memory error. + + Don't call directly this function, + but use _Py_HASHTABLE_SET() and _Py_HASHTABLE_SET_NODATA() macros */ PyAPI_FUNC(int) _Py_hashtable_set( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ + _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + + +/* Get an entry. + Return NULL if the key does not exist. + + Don't call directly this function, but use _Py_HASHTABLE_GET_ENTRY() + macro */ +PyAPI_FUNC(_Py_hashtable_entry_t*) _Py_hashtable_get_entry( + _Py_hashtable_t *ht, + size_t key_size, + const void *pkey); + +#define _Py_HASHTABLE_GET_ENTRY(TABLE, KEY) \ + _Py_hashtable_get_entry(TABLE, sizeof(KEY), &(KEY)) + + +/* Get data from an entry. Copy entry data into data and return 1 if the entry + exists, return 0 if the entry does not exist. + + Don't call directly this function, but use _Py_HASHTABLE_GET() macro */ PyAPI_FUNC(int) _Py_hashtable_get( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); + +#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ + _Py_hashtable_get(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) + + +/* Don't call directly this function, but use _Py_HASHTABLE_POP() macro */ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_t *ht, - const void *key, - void *data, - size_t data_size); -PyAPI_FUNC(void) _Py_hashtable_delete( - _Py_hashtable_t *ht, - const void *key); + size_t key_size, + const void *pkey, + size_t data_size, + void *data); -#define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) +#define _Py_HASHTABLE_POP(TABLE, KEY, DATA) \ + _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) -#define _Py_HASHTABLE_GET(TABLE, KEY, DATA) \ - _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) -#endif /* Py_LIMITED_API */ +/* Delete an entry. + + WARNING: The entry must exist. */ +PyAPI_FUNC(void) _Py_hashtable_delete( + _Py_hashtable_t *ht, + size_t key_size, + const void *pkey); +#endif /* Py_LIMITED_API */ #endif diff --git a/Python/marshal.c b/Python/marshal.c index 7a4b9d2..83a1885 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -263,10 +263,10 @@ w_ref(PyObject *v, char *flag, WFILE *p) if (Py_REFCNT(v) == 1) return 0; - entry = _Py_hashtable_get_entry(p->hashtable, v); + entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, &w, sizeof(w), entry); + _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w); /* we don't store "long" indices in the dict */ assert(0 <= w && w <= 0x7fffffff); w_byte(TYPE_REF, p); @@ -571,7 +571,8 @@ static int w_init_refs(WFILE *wf, int version) { if (version >= 3) { - wf->hashtable = _Py_hashtable_new(sizeof(int), _Py_hashtable_hash_ptr, + wf->hashtable = _Py_hashtable_new(sizeof(PyObject *), sizeof(int), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (wf->hashtable == NULL) { PyErr_NoMemory(); @@ -582,9 +583,13 @@ w_init_refs(WFILE *wf, int version) } static int -w_decref_entry(_Py_hashtable_entry_t *entry, void *Py_UNUSED(data)) +w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, + void *Py_UNUSED(data)) { - Py_XDECREF(entry->key); + PyObject *entry_key; + + _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key); + Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From b32a7edb22805daa28f486271ca5052e1ad87633 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 21 Mar 2016 23:05:08 +0100 Subject: Issue #26588: Fix compilation warning on Windows --- Modules/_tracemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6799eb6..3c5319b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -247,7 +247,7 @@ raw_free(void *ptr) static Py_uhash_t hashtable_hash_traceback(size_t key_size, const void *pkey) { - const traceback_t *traceback; + traceback_t *traceback; _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); return traceback->hash; -- cgit v0.12 From c9553876ae88b7f1494cff826c8f7a08c2ac5614 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:13:01 +0100 Subject: Simplify implementation of hashtable.c Issue #26588: Remove copy_data, free_data and get_data_size callbacks from hashtable.h. These callbacks are not used in Python and makes the code more complex. Remove also the _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P() macro which uses an unsafe pointer dereference (can cause memory alignment issue). Replace the macro usage with _Py_HASHTABLE_ENTRY_READ_DATA() which is implemented with the safe memcpy() function. --- Modules/_tracemalloc.c | 14 +++++++------ Modules/hashtable.c | 55 ++++++++------------------------------------------ Modules/hashtable.h | 14 +------------ 3 files changed, 17 insertions(+), 66 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 3c5319b..c48dd08 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -220,16 +220,15 @@ hashtable_compare_unicode(size_t key_size, const void *pkey, return key == entry_key; } -static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; - static _Py_hashtable_t * hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func) { + _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; return _Py_hashtable_new_full(key_size, data_size, 0, hash_func, compare_func, - NULL, NULL, NULL, &hashtable_alloc); + &hashtable_alloc); } static void* @@ -1120,7 +1119,8 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, _Py_hashtable_entry_t *entry, void *user_data) { - PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(tracebacks, entry); + PyObject *obj; + _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, sizeof(obj), &obj); Py_DECREF(obj); return 0; } @@ -1151,7 +1151,8 @@ py_tracemalloc_get_traces(PyObject *self, PyObject *obj) /* the traceback hash table is used temporarily to intern traceback tuple of (filename, lineno) tuples */ - get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), sizeof(PyObject *), + get_traces.tracebacks = hashtable_new(sizeof(traceback_t *), + sizeof(PyObject *), _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); if (get_traces.tracebacks == NULL) { @@ -1186,8 +1187,9 @@ finally: tracemalloc_pyobject_decref_cb, NULL); _Py_hashtable_destroy(get_traces.tracebacks); } - if (get_traces.traces != NULL) + if (get_traces.traces != NULL) { _Py_hashtable_destroy(get_traces.traces); + } return get_traces.list; } diff --git a/Modules/hashtable.c b/Modules/hashtable.c index d33f0d7..7094b95 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -128,9 +128,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, - _Py_hashtable_copy_data_func copy_data_func, - _Py_hashtable_free_data_func free_data_func, - _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator) { _Py_hashtable_t *ht; @@ -163,9 +160,6 @@ _Py_hashtable_new_full(size_t key_size, size_t data_size, ht->hash_func = hash_func; ht->compare_func = compare_func; - ht->copy_data_func = copy_data_func; - ht->free_data_func = free_data_func; - ht->get_data_size_func = get_data_size_func; ht->alloc = alloc; return ht; } @@ -179,7 +173,7 @@ _Py_hashtable_new(size_t key_size, size_t data_size, return _Py_hashtable_new_full(key_size, data_size, HASHTABLE_MIN_SIZE, hash_func, compare_func, - NULL, NULL, NULL, NULL); + NULL); } @@ -187,7 +181,6 @@ size_t _Py_hashtable_size(_Py_hashtable_t *ht) { size_t size; - size_t hv; size = sizeof(_Py_hashtable_t); @@ -197,19 +190,6 @@ _Py_hashtable_size(_Py_hashtable_t *ht) /* entries */ size += ht->entries * HASHTABLE_ITEM_SIZE(ht); - /* data linked from entries */ - if (ht->get_data_size_func) { - for (hv = 0; hv < ht->num_buckets; hv++) { - _Py_hashtable_entry_t *entry; - - for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { - void *data; - - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry); - size += ht->get_data_size_func(data); - } - } - } return size; } @@ -318,7 +298,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, int _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, - size_t data_size, void *data) + size_t data_size, const void *data) { Py_uhash_t key_hash; size_t index; @@ -380,7 +360,6 @@ _Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, size_t data_size, void *data) { assert(data != NULL); - assert(ht->free_data_func == NULL); return _Py_hashtable_pop_entry(ht, key_size, pkey, data, data_size); } @@ -470,8 +449,6 @@ _Py_hashtable_clear(_Py_hashtable_t *ht) for (i=0; i < ht->num_buckets; i++) { for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { next = ENTRY_NEXT(entry); - if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); } _Py_slist_init(&ht->buckets[i]); @@ -490,8 +467,6 @@ _Py_hashtable_destroy(_Py_hashtable_t *ht) _Py_slist_item_t *entry = ht->buckets[i].head; while (entry) { _Py_slist_item_t *entry_next = entry->next; - if (ht->free_data_func) - ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ht, entry)); ht->alloc.free(entry); entry = entry_next; } @@ -511,35 +486,21 @@ _Py_hashtable_copy(_Py_hashtable_t *src) _Py_hashtable_entry_t *entry; size_t bucket; int err; - void *data, *new_data; dst = _Py_hashtable_new_full(key_size, data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, + src->compare_func, + &src->alloc); if (dst == NULL) return NULL; for (bucket=0; bucket < src->num_buckets; bucket++) { entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { - if (src->copy_data_func) { - data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(src, entry); - new_data = src->copy_data_func(data); - if (new_data != NULL) - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, &new_data); - else - err = 1; - } - else { - data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, - _Py_HASHTABLE_ENTRY_KEY(entry), - data_size, data); - } + const void *pkey = _Py_HASHTABLE_ENTRY_KEY(entry); + const void *data = _Py_HASHTABLE_ENTRY_DATA(src, entry); + err = _Py_hashtable_set(dst, key_size, pkey, data_size, data); if (err) { _Py_hashtable_destroy(dst); return NULL; diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 6eb5737..eede038 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -35,9 +35,6 @@ typedef struct { #define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) -#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(TABLE, ENTRY) \ - (*(void **)_Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY)) - /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ #define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ @@ -66,9 +63,6 @@ typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, typedef int (*_Py_hashtable_compare_func) (size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he); -typedef void* (*_Py_hashtable_copy_data_func)(void *data); -typedef void (*_Py_hashtable_free_data_func)(void *data); -typedef size_t (*_Py_hashtable_get_data_size_func)(void *data); typedef struct { /* allocate a memory block */ @@ -90,9 +84,6 @@ typedef struct { _Py_hashtable_hash_func hash_func; _Py_hashtable_compare_func compare_func; - _Py_hashtable_copy_data_func copy_data_func; - _Py_hashtable_free_data_func free_data_func; - _Py_hashtable_get_data_size_func get_data_size_func; _Py_hashtable_allocator_t alloc; } _Py_hashtable_t; @@ -119,9 +110,6 @@ PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( size_t init_size, _Py_hashtable_hash_func hash_func, _Py_hashtable_compare_func compare_func, - _Py_hashtable_copy_data_func copy_data_func, - _Py_hashtable_free_data_func free_data_func, - _Py_hashtable_get_data_size_func get_data_size_func, _Py_hashtable_allocator_t *allocator); PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); @@ -155,7 +143,7 @@ PyAPI_FUNC(int) _Py_hashtable_set( size_t key_size, const void *pkey, size_t data_size, - void *data); + const void *data); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) -- cgit v0.12 From 58100059acb89179530036b4649f91cc679ea12b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:25:04 +0100 Subject: Remove _Py_hashtable_delete() Issue #26588: Remove _Py_hashtable_delete() from hashtable.h since the function is not used. Keep the C code in hashtable.c as commented code if someone needs it later. --- Modules/hashtable.c | 3 +++ Modules/hashtable.h | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 7094b95..bb20cce 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -364,6 +364,8 @@ _Py_hashtable_pop(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } +/* Code commented since the function is not needed in Python */ +#if 0 void _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) { @@ -374,6 +376,7 @@ _Py_hashtable_delete(_Py_hashtable_t *ht, size_t key_size, const void *pkey) (void)_Py_hashtable_pop_entry(ht, key_size, pkey, NULL, 0); #endif } +#endif int diff --git a/Modules/hashtable.h b/Modules/hashtable.h index eede038..41542d2 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -193,13 +193,5 @@ PyAPI_FUNC(int) _Py_hashtable_pop( _Py_hashtable_pop(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) -/* Delete an entry. - - WARNING: The entry must exist. */ -PyAPI_FUNC(void) _Py_hashtable_delete( - _Py_hashtable_t *ht, - size_t key_size, - const void *pkey); - #endif /* Py_LIMITED_API */ #endif -- cgit v0.12 From e492ae50e251c4fcd48bc37b1eaa4821894f1fdb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 12:58:23 +0100 Subject: tracemalloc now supports domains Issue #26588: * The _tracemalloc now supports tracing memory allocations of multiple address spaces (domains). * Add domain parameter to tracemalloc_add_trace() and tracemalloc_remove_trace(). * tracemalloc_add_trace() now starts by removing the previous trace, if any. * _tracemalloc._get_traces() now returns a list of (domain, size, traceback_frames): the domain is new. * Add tracemalloc.DomainFilter * tracemalloc.Filter: add an optional domain parameter to the constructor and a domain attribute * Sublte change: use Py_uintptr_t rather than void* in the traces key. * Add tracemalloc_config.use_domain, currently hardcoded to 1 --- Doc/library/tracemalloc.rst | 45 +++++++- Lib/test/test_tracemalloc.py | 104 ++++++++++++----- Lib/tracemalloc.py | 73 +++++++++--- Misc/NEWS | 3 + Modules/_tracemalloc.c | 267 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 388 insertions(+), 104 deletions(-) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 5feb2d9..9d1cb17 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -355,10 +355,32 @@ Functions See also the :func:`get_object_traceback` function. +DomainFilter +^^^^^^^^^^^^ + +.. class:: DomainFilter(inclusive: bool, domain: int) + + Filter traces of memory blocks by their address space (domain). + + .. versionadded:: 3.6 + + .. attribute:: inclusive + + If *inclusive* is ``True`` (include), match memory blocks allocated + in the address space :attr:`domain`. + + If *inclusive* is ``False`` (exclude), match memory blocks not allocated + in the address space :attr:`domain`. + + .. attribute:: domain + + Address space of a memory block (``int``). Read-only property. + + Filter ^^^^^^ -.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False) +.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False, domain: int=None) Filter on traces of memory blocks. @@ -378,9 +400,17 @@ Filter .. versionchanged:: 3.5 The ``'.pyo'`` file extension is no longer replaced with ``'.py'``. + .. versionchanged:: 3.6 + Added the :attr:`domain` attribute. + + + .. attribute:: domain + + Address space of a memory block (``int`` or ``None``). + .. attribute:: inclusive - If *inclusive* is ``True`` (include), only trace memory blocks allocated + If *inclusive* is ``True`` (include), only match memory blocks allocated in a file with a name matching :attr:`filename_pattern` at line number :attr:`lineno`. @@ -395,7 +425,7 @@ Filter .. attribute:: filename_pattern - Filename pattern of the filter (``str``). + Filename pattern of the filter (``str``). Read-only property. .. attribute:: all_frames @@ -458,14 +488,17 @@ Snapshot .. method:: filter_traces(filters) Create a new :class:`Snapshot` instance with a filtered :attr:`traces` - sequence, *filters* is a list of :class:`Filter` instances. If *filters* - is an empty list, return a new :class:`Snapshot` instance with a copy of - the traces. + sequence, *filters* is a list of :class:`DomainFilter` and + :class:`Filter` instances. If *filters* is an empty list, return a new + :class:`Snapshot` instance with a copy of the traces. All inclusive filters are applied at once, a trace is ignored if no inclusive filters match it. A trace is ignored if at least one exclusive filter matches it. + .. versionchanged:: 3.6 + :class:`DomainFilter` instances are now also accepted in *filters*. + .. classmethod:: load(filename) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index f65e361..7b92b87 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -37,28 +37,31 @@ def allocate_bytes(size): def create_snapshots(): traceback_limit = 2 + # _tracemalloc._get_traces() returns a list of (domain, size, + # traceback_frames) tuples. traceback_frames is a tuple of (filename, + # line_number) tuples. raw_traces = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), - (66, (('b.py', 1),)), + (2, 66, (('b.py', 1),)), - (7, (('', 0),)), + (3, 7, (('', 0),)), ] snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit) raw_traces2 = [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (5000, (('a.py', 5), ('b.py', 4))), + (2, 2, (('a.py', 5), ('b.py', 4))), + (2, 5000, (('a.py', 5), ('b.py', 4))), - (400, (('c.py', 578),)), + (4, 400, (('c.py', 578),)), ] snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit) @@ -126,7 +129,7 @@ class TestTracemallocEnabled(unittest.TestCase): def find_trace(self, traces, traceback): for trace in traces: - if trace[1] == traceback._frames: + if trace[2] == traceback._frames: return trace self.fail("trace not found") @@ -140,7 +143,7 @@ class TestTracemallocEnabled(unittest.TestCase): trace = self.find_trace(traces, obj_traceback) self.assertIsInstance(trace, tuple) - size, traceback = trace + domain, size, traceback = trace self.assertEqual(size, obj_size) self.assertEqual(traceback, obj_traceback._frames) @@ -167,9 +170,8 @@ class TestTracemallocEnabled(unittest.TestCase): trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) - size1, traceback1 = trace1 - size2, traceback2 = trace2 - self.assertEqual(traceback2, traceback1) + domain1, size1, traceback1 = trace1 + domain2, size2, traceback2 = trace2 self.assertIs(traceback2, traceback1) def test_get_traced_memory(self): @@ -292,7 +294,7 @@ class TestSnapshot(unittest.TestCase): maxDiff = 4000 def test_create_snapshot(self): - raw_traces = [(5, (('a.py', 2),))] + raw_traces = [(0, 5, (('a.py', 2),))] with contextlib.ExitStack() as stack: stack.enter_context(patch.object(tracemalloc, 'is_tracing', @@ -322,11 +324,11 @@ class TestSnapshot(unittest.TestCase): # exclude b.py snapshot3 = snapshot.filter_traces((filter1,)) self.assertEqual(snapshot3.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), - (7, (('', 0),)), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (3, 7, (('', 0),)), ]) # filter_traces() must not touch the original snapshot @@ -335,10 +337,10 @@ class TestSnapshot(unittest.TestCase): # only include two lines of a.py snapshot4 = snapshot3.filter_traces((filter2, filter3)) self.assertEqual(snapshot4.traces._traces, [ - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (10, (('a.py', 2), ('b.py', 4))), - (2, (('a.py', 5), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), ]) # No filter: just duplicate the snapshot @@ -349,6 +351,54 @@ class TestSnapshot(unittest.TestCase): self.assertRaises(TypeError, snapshot.filter_traces, filter1) + def test_filter_traces_domain(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.Filter(False, "a.py", domain=1) + filter2 = tracemalloc.Filter(True, "a.py", domain=1) + + original_traces = list(snapshot.traces._traces) + + # exclude a.py of domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('', 0),)), + ]) + + # include domain 1 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + (3, 7, (('', 0),)), + ]) + + def test_filter_traces_domain_filter(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.DomainFilter(False, domain=3) + filter2 = tracemalloc.DomainFilter(True, domain=3) + + # exclude domain 2 + snapshot3 = snapshot.filter_traces((filter1,)) + self.assertEqual(snapshot3.traces._traces, [ + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (0, 10, (('a.py', 2), ('b.py', 4))), + (1, 2, (('a.py', 5), ('b.py', 4))), + (2, 66, (('b.py', 1),)), + ]) + + # include domain 2 + snapshot3 = snapshot.filter_traces((filter2,)) + self.assertEqual(snapshot3.traces._traces, [ + (3, 7, (('', 0),)), + ]) + def test_snapshot_group_by_line(self): snapshot, snapshot2 = create_snapshots() tb_0 = traceback_lineno('', 0) diff --git a/Lib/tracemalloc.py b/Lib/tracemalloc.py index 6288da8..75b3918 100644 --- a/Lib/tracemalloc.py +++ b/Lib/tracemalloc.py @@ -244,17 +244,21 @@ class Trace: __slots__ = ("_trace",) def __init__(self, trace): - # trace is a tuple: (size, traceback), see Traceback constructor - # for the format of the traceback tuple + # trace is a tuple: (domain: int, size: int, traceback: tuple). + # See Traceback constructor for the format of the traceback tuple. self._trace = trace @property - def size(self): + def domain(self): return self._trace[0] @property + def size(self): + return self._trace[1] + + @property def traceback(self): - return Traceback(self._trace[1]) + return Traceback(self._trace[2]) def __eq__(self, other): return (self._trace == other._trace) @@ -266,8 +270,8 @@ class Trace: return "%s: %s" % (self.traceback, _format_size(self.size, False)) def __repr__(self): - return ("" - % (_format_size(self.size, False), self.traceback)) + return ("" + % (self.domain, _format_size(self.size, False), self.traceback)) class _Traces(Sequence): @@ -302,19 +306,29 @@ def _normalize_filename(filename): return filename -class Filter: +class BaseFilter: + def __init__(self, inclusive): + self.inclusive = inclusive + + def _match(self, trace): + raise NotImplementedError + + +class Filter(BaseFilter): def __init__(self, inclusive, filename_pattern, - lineno=None, all_frames=False): + lineno=None, all_frames=False, domain=None): + super().__init__(inclusive) self.inclusive = inclusive self._filename_pattern = _normalize_filename(filename_pattern) self.lineno = lineno self.all_frames = all_frames + self.domain = domain @property def filename_pattern(self): return self._filename_pattern - def __match_frame(self, filename, lineno): + def _match_frame_impl(self, filename, lineno): filename = _normalize_filename(filename) if not fnmatch.fnmatch(filename, self._filename_pattern): return False @@ -324,11 +338,11 @@ class Filter: return (lineno == self.lineno) def _match_frame(self, filename, lineno): - return self.__match_frame(filename, lineno) ^ (not self.inclusive) + return self._match_frame_impl(filename, lineno) ^ (not self.inclusive) def _match_traceback(self, traceback): if self.all_frames: - if any(self.__match_frame(filename, lineno) + if any(self._match_frame_impl(filename, lineno) for filename, lineno in traceback): return self.inclusive else: @@ -337,6 +351,30 @@ class Filter: filename, lineno = traceback[0] return self._match_frame(filename, lineno) + def _match(self, trace): + domain, size, traceback = trace + res = self._match_traceback(traceback) + if self.domain is not None: + if self.inclusive: + return res and (domain == self.domain) + else: + return res or (domain != self.domain) + return res + + +class DomainFilter(BaseFilter): + def __init__(self, inclusive, domain): + super().__init__(inclusive) + self._domain = domain + + @property + def domain(self): + return self._domain + + def _match(self, trace): + domain, size, traceback = trace + return (domain == self.domain) ^ (not self.inclusive) + class Snapshot: """ @@ -365,13 +403,12 @@ class Snapshot: return pickle.load(fp) def _filter_trace(self, include_filters, exclude_filters, trace): - traceback = trace[1] if include_filters: - if not any(trace_filter._match_traceback(traceback) + if not any(trace_filter._match(trace) for trace_filter in include_filters): return False if exclude_filters: - if any(not trace_filter._match_traceback(traceback) + if any(not trace_filter._match(trace) for trace_filter in exclude_filters): return False return True @@ -379,8 +416,8 @@ class Snapshot: def filter_traces(self, filters): """ Create a new Snapshot instance with a filtered traces sequence, filters - is a list of Filter instances. If filters is an empty list, return a - new Snapshot instance with a copy of the traces. + is a list of Filter or DomainFilter instances. If filters is an empty + list, return a new Snapshot instance with a copy of the traces. """ if not isinstance(filters, Iterable): raise TypeError("filters must be a list of filters, not %s" @@ -412,7 +449,7 @@ class Snapshot: tracebacks = {} if not cumulative: for trace in self.traces._traces: - size, trace_traceback = trace + domain, size, trace_traceback = trace try: traceback = tracebacks[trace_traceback] except KeyError: @@ -433,7 +470,7 @@ class Snapshot: else: # cumulative statistics for trace in self.traces._traces: - size, trace_traceback = trace + domain, size, trace_traceback = trace for frame in trace_traceback: try: traceback = tracebacks[frame] diff --git a/Misc/NEWS b/Misc/NEWS index e6d69a2..841f8a0 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,9 @@ Core and Builtins Library ------- +- Issue #26588: The _tracemalloc now supports tracing memory allocations of + multiple address spaces (domains). + - Issue #24266: Ctrl+C during Readline history search now cancels the search mode when compiled with Readline 7. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index c48dd08..784157f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -39,7 +39,11 @@ static struct { /* limit of the number of frames in a traceback, 1 by default. Variable protected by the GIL. */ int max_nframe; -} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1}; + + /* use domain in trace key? + Variable protected by the GIL. */ + int use_domain; +} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without @@ -54,10 +58,23 @@ static PyThread_type_lock tables_lock; # define TABLES_UNLOCK() #endif + +#define DEFAULT_DOMAIN 0 + +typedef unsigned int domain_t; + +/* Pack the frame_t structure to reduce the memory footprint. */ +typedef struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +{ + Py_uintptr_t ptr; + domain_t domain; +} pointer_t; + /* Pack the frame_t structure to reduce the memory footprint on 64-bit - architectures: 12 bytes instead of 16. This optimization might produce - SIGBUS on architectures not supporting unaligned memory accesses (64-bit - MIPS CPU?): on such architecture, the structure must not be packed. */ + architectures: 12 bytes instead of 16. */ typedef struct #ifdef __GNUC__ __attribute__((packed)) @@ -71,6 +88,7 @@ _declspec(align(4)) unsigned int lineno; } frame_t; + typedef struct { Py_uhash_t hash; int nframe; @@ -83,6 +101,7 @@ typedef struct { #define MAX_NFRAME \ ((INT_MAX - (int)sizeof(traceback_t)) / (int)sizeof(frame_t) + 1) + static PyObject *unknown_filename = NULL; static traceback_t tracemalloc_empty_traceback; @@ -95,6 +114,7 @@ typedef struct { traceback_t *traceback; } trace_t; + /* Size in bytes of currently traced memory. Protected by TABLES_LOCK(). */ static size_t tracemalloc_traced_memory = 0; @@ -121,6 +141,7 @@ static _Py_hashtable_t *tracemalloc_tracebacks = NULL; Protected by TABLES_LOCK(). */ static _Py_hashtable_t *tracemalloc_traces = NULL; + #ifdef TRACE_DEBUG static void tracemalloc_error(const char *format, ...) @@ -135,6 +156,7 @@ tracemalloc_error(const char *format, ...) } #endif + #if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) #define REENTRANT_THREADLOCAL @@ -196,6 +218,7 @@ set_reentrant(int reentrant) } #endif + static Py_uhash_t hashtable_hash_pyobject(size_t key_size, const void *pkey) { @@ -205,21 +228,53 @@ hashtable_hash_pyobject(size_t key_size, const void *pkey) return PyObject_Hash(obj); } + static int hashtable_compare_unicode(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *entry) { - PyObject *key, *entry_key; + PyObject *key1, *key2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, entry_key); + _Py_HASHTABLE_READ_KEY(key_size, pkey, key1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, key2); - if (key != NULL && entry_key != NULL) - return (PyUnicode_Compare(key, entry_key) == 0); + if (key1 != NULL && key2 != NULL) + return (PyUnicode_Compare(key1, key2) == 0); else - return key == entry_key; + return key1 == key2; +} + + +static Py_uhash_t +hashtable_hash_pointer_t(size_t key_size, const void *pkey) +{ + pointer_t ptr; + Py_uhash_t hash; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr); + + hash = (Py_uhash_t)_Py_HashPointer((void*)ptr.ptr); + hash ^= ptr.domain; + return hash; +} + + +int +hashtable_compare_pointer_t(size_t key_size, const void *pkey, + const _Py_hashtable_entry_t *entry) +{ + pointer_t ptr1, ptr2; + + _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr1); + _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, ptr2); + + /* compare pointer before domain, because pointer is more likely to be + different */ + return (ptr1.ptr == ptr2.ptr && ptr1.domain == ptr2.domain); + } + static _Py_hashtable_t * hashtable_new(size_t key_size, size_t data_size, _Py_hashtable_hash_func hash_func, @@ -231,6 +286,7 @@ hashtable_new(size_t key_size, size_t data_size, &hashtable_alloc); } + static void* raw_malloc(size_t size) { @@ -243,6 +299,7 @@ raw_free(void *ptr) allocators.raw.free(allocators.raw.ctx, ptr); } + static Py_uhash_t hashtable_hash_traceback(size_t key_size, const void *pkey) { @@ -252,6 +309,7 @@ hashtable_hash_traceback(size_t key_size, const void *pkey) return traceback->hash; } + static int hashtable_compare_traceback(size_t key_size, const void *pkey, const _Py_hashtable_entry_t *he) @@ -281,6 +339,7 @@ hashtable_compare_traceback(size_t key_size, const void *pkey, return 1; } + static void tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) { @@ -353,6 +412,7 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) frame->filename = filename; } + static Py_uhash_t traceback_hash(traceback_t *traceback) { @@ -377,6 +437,7 @@ traceback_hash(traceback_t *traceback) return x; } + static void traceback_get_frames(traceback_t *traceback) { @@ -404,6 +465,7 @@ traceback_get_frames(traceback_t *traceback) } } + static traceback_t * traceback_new(void) { @@ -455,41 +517,72 @@ traceback_new(void) return traceback; } + +static void +tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) +{ + trace_t trace; + int removed; + + if (tracemalloc_config.use_domain) { + pointer_t key = {ptr, domain}; + removed = _Py_HASHTABLE_POP(tracemalloc_traces, key, trace); + } + else { + removed = _Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace); + } + if (!removed) { + return; + } + + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; +} + +#define REMOVE_TRACE(ptr) \ + tracemalloc_remove_trace(DEFAULT_DOMAIN, (Py_uintptr_t)(ptr)) + + static int -tracemalloc_add_trace(void *ptr, size_t size) +tracemalloc_add_trace(domain_t domain, Py_uintptr_t ptr, size_t size) { traceback_t *traceback; trace_t trace; int res; + /* first, remove the previous trace (if any) */ + tracemalloc_remove_trace(domain, ptr); + traceback = traceback_new(); - if (traceback == NULL) + if (traceback == NULL) { return -1; + } trace.size = size; trace.traceback = traceback; - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); - if (res == 0) { - assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); - tracemalloc_traced_memory += size; - if (tracemalloc_traced_memory > tracemalloc_peak_traced_memory) - tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + if (tracemalloc_config.use_domain) { + pointer_t key = {ptr, domain}; + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + } + else { + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + } + + if (res != 0) { + return res; } - return res; + assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); + tracemalloc_traced_memory += size; + if (tracemalloc_traced_memory > tracemalloc_peak_traced_memory) + tracemalloc_peak_traced_memory = tracemalloc_traced_memory; + return 0; } -static void -tracemalloc_remove_trace(void *ptr) -{ - trace_t trace; +#define ADD_TRACE(ptr, size) \ + tracemalloc_add_trace(DEFAULT_DOMAIN, (Py_uintptr_t)(ptr), size) - if (_Py_HASHTABLE_POP(tracemalloc_traces, ptr, trace)) { - assert(tracemalloc_traced_memory >= trace.size); - tracemalloc_traced_memory -= trace.size; - } -} static void* tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) @@ -507,7 +600,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return NULL; TABLES_LOCK(); - if (tracemalloc_add_trace(ptr, nelem * elsize) < 0) { + if (ADD_TRACE(ptr, nelem * elsize) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr); @@ -517,6 +610,7 @@ tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) { @@ -531,9 +625,9 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (ADD_TRACE(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to the caller, because realloc() may already have shrinked the memory block and so removed bytes. @@ -551,7 +645,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* new allocation */ TABLES_LOCK(); - if (tracemalloc_add_trace(ptr2, new_size) < 0) { + if (ADD_TRACE(ptr2, new_size) < 0) { /* Failed to allocate a trace for the new memory block */ TABLES_UNLOCK(); alloc->free(alloc->ctx, ptr2); @@ -562,6 +656,7 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) return ptr2; } + static void tracemalloc_free(void *ctx, void *ptr) { @@ -576,10 +671,11 @@ tracemalloc_free(void *ctx, void *ptr) alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } + static void* tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) { @@ -604,18 +700,21 @@ tracemalloc_alloc_gil(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_malloc_gil(void *ctx, size_t size) { return tracemalloc_alloc_gil(0, ctx, 1, size); } + static void* tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) { return tracemalloc_alloc_gil(1, ctx, nelem, elsize); } + static void* tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) { @@ -631,7 +730,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } return ptr2; @@ -648,6 +747,7 @@ tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) return ptr2; } + #ifdef TRACE_RAW_MALLOC static void* tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) @@ -682,18 +782,21 @@ tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) return ptr; } + static void* tracemalloc_raw_malloc(void *ctx, size_t size) { return tracemalloc_raw_alloc(0, ctx, 1, size); } + static void* tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) { return tracemalloc_raw_alloc(1, ctx, nelem, elsize); } + static void* tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) { @@ -710,7 +813,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) if (ptr2 != NULL && ptr != NULL) { TABLES_LOCK(); - tracemalloc_remove_trace(ptr); + REMOVE_TRACE(ptr); TABLES_UNLOCK(); } return ptr2; @@ -734,6 +837,7 @@ tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) } #endif /* TRACE_RAW_MALLOC */ + static int tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *user_data) @@ -745,6 +849,7 @@ tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, return 0; } + static int traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, void *user_data) @@ -756,6 +861,7 @@ traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, return 0; } + /* reentrant flag must be set to call this function and GIL must be held */ static void tracemalloc_clear_traces(void) @@ -782,6 +888,7 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } + static int tracemalloc_init(void) { @@ -826,9 +933,18 @@ tracemalloc_init(void) hashtable_hash_traceback, hashtable_compare_traceback); - tracemalloc_traces = hashtable_new(sizeof(void*), sizeof(trace_t), - _Py_hashtable_hash_ptr, - _Py_hashtable_compare_direct); + if (tracemalloc_config.use_domain) { + tracemalloc_traces = hashtable_new(sizeof(pointer_t), + sizeof(trace_t), + hashtable_hash_pointer_t, + hashtable_compare_pointer_t); + } + else { + tracemalloc_traces = hashtable_new(sizeof(Py_uintptr_t), + sizeof(trace_t), + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + } if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL || tracemalloc_traces == NULL) { @@ -856,6 +972,7 @@ tracemalloc_init(void) return 0; } + static void tracemalloc_deinit(void) { @@ -884,6 +1001,7 @@ tracemalloc_deinit(void) Py_XDECREF(unknown_filename); } + static int tracemalloc_start(int max_nframe) { @@ -941,6 +1059,7 @@ tracemalloc_start(int max_nframe) return 0; } + static void tracemalloc_stop(void) { @@ -974,6 +1093,7 @@ PyDoc_STRVAR(tracemalloc_is_tracing_doc, "True if the tracemalloc module is tracing Python memory allocations,\n" "False otherwise."); + static PyObject* py_tracemalloc_is_tracing(PyObject *self) { @@ -985,6 +1105,7 @@ PyDoc_STRVAR(tracemalloc_clear_traces_doc, "\n" "Clear traces of memory blocks allocated by Python."); + static PyObject* py_tracemalloc_clear_traces(PyObject *self) { @@ -998,6 +1119,7 @@ py_tracemalloc_clear_traces(PyObject *self) Py_RETURN_NONE; } + static PyObject* frame_to_pyobject(frame_t *frame) { @@ -1020,6 +1142,7 @@ frame_to_pyobject(frame_t *frame) return frame_obj; } + static PyObject* traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) { @@ -1058,33 +1181,43 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) return frames; } + static PyObject* -trace_to_pyobject(trace_t *trace, _Py_hashtable_t *intern_tracebacks) +trace_to_pyobject(domain_t domain, trace_t *trace, + _Py_hashtable_t *intern_tracebacks) { PyObject *trace_obj = NULL; - PyObject *size, *traceback; + PyObject *obj; - trace_obj = PyTuple_New(2); + trace_obj = PyTuple_New(3); if (trace_obj == NULL) return NULL; - size = PyLong_FromSize_t(trace->size); - if (size == NULL) { + obj = PyLong_FromSize_t(domain); + if (obj == NULL) { Py_DECREF(trace_obj); return NULL; } - PyTuple_SET_ITEM(trace_obj, 0, size); + PyTuple_SET_ITEM(trace_obj, 0, obj); - traceback = traceback_to_pyobject(trace->traceback, intern_tracebacks); - if (traceback == NULL) { + obj = PyLong_FromSize_t(trace->size); + if (obj == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 1, obj); + + obj = traceback_to_pyobject(trace->traceback, intern_tracebacks); + if (obj == NULL) { Py_DECREF(trace_obj); return NULL; } - PyTuple_SET_ITEM(trace_obj, 1, traceback); + PyTuple_SET_ITEM(trace_obj, 2, obj); return trace_obj; } + typedef struct { _Py_hashtable_t *traces; _Py_hashtable_t *tracebacks; @@ -1096,13 +1229,22 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr void *user_data) { get_traces_t *get_traces = user_data; + domain_t domain; trace_t *trace; PyObject *tracemalloc_obj; int res; + if (tracemalloc_config.use_domain) { + pointer_t key; + _Py_HASHTABLE_ENTRY_READ_KEY(traces->key_size, entry, key); + domain = key.domain; + } + else { + domain = DEFAULT_DOMAIN; + } trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); - tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); + tracemalloc_obj = trace_to_pyobject(domain, trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) return 1; @@ -1114,6 +1256,7 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr return 0; } + static int tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, _Py_hashtable_entry_t *entry, @@ -1125,6 +1268,7 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, return 0; } + PyDoc_STRVAR(tracemalloc_get_traces_doc, "_get_traces() -> list\n" "\n" @@ -1194,8 +1338,9 @@ finally: return get_traces.list; } + static traceback_t* -tracemalloc_get_traceback(const void *ptr) +tracemalloc_get_traceback(domain_t domain, const void *ptr) { trace_t trace; int found; @@ -1204,7 +1349,13 @@ tracemalloc_get_traceback(const void *ptr) return NULL; TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + if (tracemalloc_config.use_domain) { + pointer_t key = {(Py_uintptr_t)ptr, domain}; + found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); + } + else { + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + } TABLES_UNLOCK(); if (!found) @@ -1213,6 +1364,7 @@ tracemalloc_get_traceback(const void *ptr) return trace.traceback; } + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1235,13 +1387,14 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); if (traceback == NULL) Py_RETURN_NONE; return traceback_to_pyobject(traceback, NULL); } + #define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) static void @@ -1262,7 +1415,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); if (traceback == NULL) return; @@ -1275,6 +1428,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) #undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" @@ -1310,6 +1464,7 @@ PyDoc_STRVAR(tracemalloc_stop_doc, "Stop tracing Python memory allocations and clear traces\n" "of memory blocks allocated by Python."); + static PyObject* py_tracemalloc_stop(PyObject *self) { @@ -1317,6 +1472,7 @@ py_tracemalloc_stop(PyObject *self) Py_RETURN_NONE; } + PyDoc_STRVAR(tracemalloc_get_traceback_limit_doc, "get_traceback_limit() -> int\n" "\n" @@ -1332,6 +1488,7 @@ py_tracemalloc_get_traceback_limit(PyObject *self) return PyLong_FromLong(tracemalloc_config.max_nframe); } + PyDoc_STRVAR(tracemalloc_get_tracemalloc_memory_doc, "get_tracemalloc_memory() -> int\n" "\n" @@ -1355,6 +1512,7 @@ tracemalloc_get_tracemalloc_memory(PyObject *self) return Py_BuildValue("N", size_obj); } + PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, "get_traced_memory() -> (int, int)\n" "\n" @@ -1380,6 +1538,7 @@ tracemalloc_get_traced_memory(PyObject *self) return Py_BuildValue("NN", size_obj, peak_size_obj); } + static PyMethodDef module_methods[] = { {"is_tracing", (PyCFunction)py_tracemalloc_is_tracing, METH_NOARGS, tracemalloc_is_tracing_doc}, @@ -1430,6 +1589,7 @@ PyInit__tracemalloc(void) return m; } + static int parse_sys_xoptions(PyObject *value) { @@ -1458,6 +1618,7 @@ parse_sys_xoptions(PyObject *value) return Py_SAFE_DOWNCAST(nframe, long, int); } + int _PyTraceMalloc_Init(void) { @@ -1516,6 +1677,7 @@ _PyTraceMalloc_Init(void) return tracemalloc_start(nframe); } + void _PyTraceMalloc_Fini(void) { @@ -1524,4 +1686,3 @@ _PyTraceMalloc_Fini(void) #endif tracemalloc_deinit(); } - -- cgit v0.12 From 10b73e17489048419b512f6710aecba62ff5b91a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 13:39:05 +0100 Subject: Add C functions _PyTraceMalloc_Track() Issue #26530: * Add C functions _PyTraceMalloc_Track() and _PyTraceMalloc_Untrack() to track memory blocks using the tracemalloc module. * Add _PyTraceMalloc_GetTraceback() to get the traceback of an object. --- Include/pymem.h | 34 +++++++++++++ Lib/test/test_tracemalloc.py | 115 +++++++++++++++++++++++++++++++++++++++++++ Misc/NEWS | 5 ++ Modules/_testcapimodule.c | 75 ++++++++++++++++++++++++++++ Modules/_tracemalloc.c | 82 +++++++++++++++++++++++++----- 5 files changed, 300 insertions(+), 11 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index b1f06ef..941e00f 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -25,6 +25,40 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif +/* Identifier of an address space (domain) in tracemalloc */ +typedef unsigned int _PyTraceMalloc_domain_t; + +/* Track an allocated memory block in the tracemalloc module. + Return 0 on success, return -1 on error (failed to allocate memory to store + the trace). + + Return -2 if tracemalloc is disabled. + + If memory block was already tracked, begin by removing the old trace. */ +PyAPI_FUNC(int) _PyTraceMalloc_Track( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr, + size_t size); + +/* Untrack an allocated memory block in the tracemalloc module. + Do nothing if the block was not tracked. + + Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) _PyTraceMalloc_Untrack( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr); + +/* Get the traceback where a memory block was allocated. + + Return a tuple of (filename: str, lineno: int) tuples. + + Return None if the tracemalloc module is disabled or if the memory block + is not tracked by tracemalloc. + + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( + _PyTraceMalloc_domain_t domain, + Py_uintptr_t ptr); #endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 7b92b87..359d9c0 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -11,9 +11,15 @@ try: import threading except ImportError: threading = None +try: + import _testcapi +except ImportError: + _testcapi = None + EMPTY_STRING_SIZE = sys.getsizeof(b'') + def get_frames(nframe, lineno_delta): frames = [] frame = sys._getframe(1) @@ -866,12 +872,121 @@ class TestCommandLine(unittest.TestCase): assert_python_ok('-X', 'tracemalloc', '-c', code) +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + maxDiff = 80 * 20 + + def setUp(self): + if tracemalloc.is_tracing(): + self.skipTest("tracemalloc must be stopped before the test") + + self.domain = 5 + self.size = 123 + self.obj = allocate_bytes(self.size)[0] + + # for the type "object", id(obj) is the address of its memory block. + # This type is not tracked by the garbage collector + self.ptr = id(self.obj) + + def tearDown(self): + tracemalloc.stop() + + def get_traceback(self): + frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) + if frames is not None: + return tracemalloc.Traceback(frames) + else: + return None + + def track(self, release_gil=False, nframe=1): + frames = get_frames(nframe, 2) + _testcapi.tracemalloc_track(self.domain, self.ptr, self.size, + release_gil) + return frames + + def untrack(self): + _testcapi.tracemalloc_untrack(self.domain, self.ptr) + + def get_traced_memory(self): + # Get the traced size in the domain + snapshot = tracemalloc.take_snapshot() + domain_filter = tracemalloc.DomainFilter(True, self.domain) + snapshot = snapshot.filter_traces([domain_filter]) + return sum(trace.size for trace in snapshot.traces) + + def check_track(self, release_gil): + nframe = 5 + tracemalloc.start(nframe) + + size = tracemalloc.get_traced_memory()[0] + + frames = self.track(release_gil, nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + self.assertEqual(self.get_traced_memory(), self.size) + + def test_track(self): + self.check_track(False) + + def test_track_without_gil(self): + # check that calling _PyTraceMalloc_Track() without holding the GIL + # works too + self.check_track(True) + + def test_track_already_tracked(self): + nframe = 5 + tracemalloc.start(nframe) + + # track a first time + self.track() + + # calling _PyTraceMalloc_Track() must remove the old trace and add + # a new trace with the new traceback + frames = self.track(nframe=nframe) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) + + def test_untrack(self): + tracemalloc.start() + + self.track() + self.assertIsNotNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), self.size) + + # untrack must remove the trace + self.untrack() + self.assertIsNone(self.get_traceback()) + self.assertEqual(self.get_traced_memory(), 0) + + # calling _PyTraceMalloc_Untrack() multiple times must not crash + self.untrack() + self.untrack() + + def test_stop_track(self): + tracemalloc.start() + tracemalloc.stop() + + with self.assertRaises(RuntimeError): + self.track() + self.assertIsNone(self.get_traceback()) + + def test_stop_untrack(self): + tracemalloc.start() + self.track() + + tracemalloc.stop() + with self.assertRaises(RuntimeError): + self.untrack() + + def test_main(): support.run_unittest( TestTracemallocEnabled, TestSnapshot, TestFilters, TestCommandLine, + TestCAPI, ) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 841f8a0..29fc65d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,11 @@ Core and Builtins Library ------- +- Issue #26530: Add C functions :c:func:`_PyTraceMalloc_Track` and + :c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the + :mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get + the traceback of an object. + - Issue #26588: The _tracemalloc now supports tracing memory allocations of multiple address spaces (domains). diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0fc7cbc..8c79485 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3675,6 +3675,78 @@ pyobject_malloc_without_gil(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + Py_ssize_t size; + int release_gil = 0; + int res; + + if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size); + Py_END_ALLOW_THREADS + } + else { + res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size); + } + + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + int res; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + res = _PyTraceMalloc_Untrack(domain, (Py_uintptr_t)ptr); + if (res < 0) { + PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{ + unsigned int domain; + PyObject *ptr_obj; + void *ptr; + + if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) + return NULL; + ptr = PyLong_AsVoidPtr(ptr_obj); + if (PyErr_Occurred()) + return NULL; + + return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3861,6 +3933,9 @@ static PyMethodDef TestMethods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, + {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, + {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, + {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 784157f..5ff1f84 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -61,8 +61,6 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -typedef unsigned int domain_t; - /* Pack the frame_t structure to reduce the memory footprint. */ typedef struct #ifdef __GNUC__ @@ -70,7 +68,7 @@ __attribute__((packed)) #endif { Py_uintptr_t ptr; - domain_t domain; + _PyTraceMalloc_domain_t domain; } pointer_t; /* Pack the frame_t structure to reduce the memory footprint on 64-bit @@ -519,11 +517,13 @@ traceback_new(void) static void -tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) +tracemalloc_remove_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int removed; + assert(tracemalloc_config.tracing); + if (tracemalloc_config.use_domain) { pointer_t key = {ptr, domain}; removed = _Py_HASHTABLE_POP(tracemalloc_traces, key, trace); @@ -544,12 +544,15 @@ tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) static int -tracemalloc_add_trace(domain_t domain, Py_uintptr_t ptr, size_t size) +tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, + size_t size) { traceback_t *traceback; trace_t trace; int res; + assert(tracemalloc_config.tracing); + /* first, remove the previous trace (if any) */ tracemalloc_remove_trace(domain, ptr); @@ -1183,7 +1186,7 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) static PyObject* -trace_to_pyobject(domain_t domain, trace_t *trace, +trace_to_pyobject(_PyTraceMalloc_domain_t domain, trace_t *trace, _Py_hashtable_t *intern_tracebacks) { PyObject *trace_obj = NULL; @@ -1229,7 +1232,7 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr void *user_data) { get_traces_t *get_traces = user_data; - domain_t domain; + _PyTraceMalloc_domain_t domain; trace_t *trace; PyObject *tracemalloc_obj; int res; @@ -1340,7 +1343,7 @@ finally: static traceback_t* -tracemalloc_get_traceback(domain_t domain, const void *ptr) +tracemalloc_get_traceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int found; @@ -1350,7 +1353,7 @@ tracemalloc_get_traceback(domain_t domain, const void *ptr) TABLES_LOCK(); if (tracemalloc_config.use_domain) { - pointer_t key = {(Py_uintptr_t)ptr, domain}; + pointer_t key = {ptr, domain}; found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace); } else { @@ -1387,7 +1390,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) Py_RETURN_NONE; @@ -1415,7 +1418,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) traceback_t *traceback; int i; - traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr); + traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) return; @@ -1686,3 +1689,60 @@ _PyTraceMalloc_Fini(void) #endif tracemalloc_deinit(); } + +int +_PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, + size_t size) +{ + int res; +#ifdef WITH_THREAD + PyGILState_STATE gil_state; +#endif + + if (!tracemalloc_config.tracing) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } + +#ifdef WITH_THREAD + gil_state = PyGILState_Ensure(); +#endif + + TABLES_LOCK(); + res = tracemalloc_add_trace(domain, ptr, size); + TABLES_UNLOCK(); + +#ifdef WITH_THREAD + PyGILState_Release(gil_state); +#endif + return res; +} + + +int +_PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{ + if (!tracemalloc_config.tracing) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } + + TABLES_LOCK(); + tracemalloc_remove_trace(domain, ptr); + TABLES_UNLOCK(); + + return 0; +} + + +PyObject* +_PyTraceMalloc_GetTraceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{ + traceback_t *traceback; + + traceback = tracemalloc_get_traceback(domain, ptr); + if (traceback == NULL) + Py_RETURN_NONE; + + return traceback_to_pyobject(traceback, NULL); +} -- cgit v0.12 From 24f949e10c4031c059118dd4b49cace16d4e3e5d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 15:14:09 +0100 Subject: regrtest: add time to output Timestamps should help to debug slow buildbots, and timeout and hang on buildbots. --- Lib/test/libregrtest/main.py | 23 ++++++++++++++++++----- Lib/test/test_regrtest.py | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 82788ad..1c99f2b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,3 +1,4 @@ +import datetime import faulthandler import os import platform @@ -7,6 +8,7 @@ import sys import sysconfig import tempfile import textwrap +import time from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, @@ -79,6 +81,7 @@ class Regrtest: self.found_garbage = [] # used to display the progress bar "[ 3/100]" + self.start_time = time.monotonic() self.test_count = '' self.test_count_width = 1 @@ -102,16 +105,24 @@ class Regrtest: self.skipped.append(test) self.resource_denieds.append(test) + def time_delta(self): + seconds = time.monotonic() - self.start_time + return datetime.timedelta(seconds=int(seconds)) + def display_progress(self, test_index, test): if self.ns.quiet: return if self.bad and not self.ns.pgo: - fmt = "[{1:{0}}{2}/{3}] {4}" + fmt = "{time} [{test_index:{count_width}}{test_count}/{nbad}] {test_name}" else: - fmt = "[{1:{0}}{2}] {4}" - print(fmt.format(self.test_count_width, test_index, - self.test_count, len(self.bad), test), - flush=True) + fmt = "{time} [{test_index:{count_width}}{test_count}] {test_name}" + line = fmt.format(count_width=self.test_count_width, + test_index=test_index, + test_count=self.test_count, + nbad=len(self.bad), + test_name=test, + time=self.time_delta()) + print(line, flush=True) def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) @@ -368,6 +379,8 @@ class Regrtest: r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) + print("Total duration: %s" % self.time_delta()) + if self.ns.runleaks: os.system("leaks %d" % os.getpid()) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 59f8c9d..18d0f46 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -351,7 +351,7 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = r'^\[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + regex = r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) -- cgit v0.12 From 84aab09421330711607671ed8bc637ea2a8e273c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 16:13:31 +0100 Subject: Issue #26588: add debug traces Try to debug random failure on buildbots. --- Lib/test/test_tracemalloc.py | 10 ++++++++++ Modules/_testcapimodule.c | 14 ++++++++++++++ Modules/_tracemalloc.c | 40 +++++++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 359d9c0..44cd539 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -88,6 +88,9 @@ def traceback_filename(filename): class TestTracemallocEnabled(unittest.TestCase): def setUp(self): + if _testcapi: + _testcapi.tracemalloc_set_debug(True) + if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -95,6 +98,8 @@ class TestTracemallocEnabled(unittest.TestCase): def tearDown(self): tracemalloc.stop() + if _testcapi: + _testcapi.tracemalloc_set_debug(False) def test_get_tracemalloc_memory(self): data = [allocate_bytes(123) for count in range(1000)] @@ -877,6 +882,9 @@ class TestCAPI(unittest.TestCase): maxDiff = 80 * 20 def setUp(self): + if _testcapi: + _testcapi.tracemalloc_set_debug(True) + if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -890,6 +898,8 @@ class TestCAPI(unittest.TestCase): def tearDown(self): tracemalloc.stop() + if _testcapi: + _testcapi.tracemalloc_set_debug(False) def get_traceback(self): frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 8c79485..f952aec 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3747,6 +3747,19 @@ tracemalloc_get_traceback(PyObject *self, PyObject *args) return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); } +PyObject* +tracemalloc_set_debug(PyObject *self, PyObject *args) +{ + int debug; + extern int tracemalloc_debug; + + if (!PyArg_ParseTuple(args, "i", &debug)) + return NULL; + + tracemalloc_debug = debug; + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3936,6 +3949,7 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, + {"tracemalloc_set_debug", tracemalloc_set_debug, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5ff1f84..551bade 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -45,6 +45,8 @@ static struct { int use_domain; } tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; +int tracemalloc_debug = 0; + #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it @@ -891,18 +893,24 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } +#define DEBUG(MSG) \ + if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li] " MSG "\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); } + static int tracemalloc_init(void) { +DEBUG("tracemalloc_init()"); if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); return -1; } - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) + if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) { +DEBUG("tracemalloc_init(): exit (already initialized)"); return 0; + } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); @@ -969,9 +977,11 @@ tracemalloc_init(void) /* Disable tracing allocations until hooks are installed. Set also the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ +DEBUG("tracemalloc_init(): set_reentrant(1)"); set_reentrant(1); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; +DEBUG("tracemalloc_init(): done"); return 0; } @@ -979,8 +989,11 @@ tracemalloc_init(void) static void tracemalloc_deinit(void) { - if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) +DEBUG("tracemalloc_deinit()"); + if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) { +DEBUG("tracemalloc_deinit(): exit (not initialized)"); return; + } tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; tracemalloc_stop(); @@ -997,11 +1010,13 @@ tracemalloc_deinit(void) } #endif +DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL PyThread_delete_key(tracemalloc_reentrant_key); #endif Py_XDECREF(unknown_filename); +DEBUG("tracemalloc_deinit(): done"); } @@ -1011,11 +1026,15 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; - if (tracemalloc_init() < 0) +DEBUG("tracemalloc_start()"); + if (tracemalloc_init() < 0) { +DEBUG("tracemalloc_start(): ERROR! init failed!"); return -1; + } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ +DEBUG("tracemalloc_start(): exit (already tracing)"); return 0; } @@ -1057,8 +1076,11 @@ tracemalloc_start(int max_nframe) /* everything is ready: start tracing Python memory allocations */ tracemalloc_config.tracing = 1; + +DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); +DEBUG("tracemalloc_start(): done"); return 0; } @@ -1066,14 +1088,18 @@ tracemalloc_start(int max_nframe) static void tracemalloc_stop(void) { - if (!tracemalloc_config.tracing) +DEBUG("tracemalloc_stop()"); + if (!tracemalloc_config.tracing) { +DEBUG("tracemalloc_stop(): exit (not tracing)"); return; + } /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; /* set the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ +DEBUG("tracemalloc_stop(): set_reentrant(1)"); set_reentrant(1); /* unregister the hook on memory allocators */ @@ -1088,6 +1114,7 @@ tracemalloc_stop(void) /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; +DEBUG("tracemalloc_stop(): done"); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, @@ -1455,8 +1482,11 @@ py_tracemalloc_start(PyObject *self, PyObject *args) } nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - if (tracemalloc_start(nframe_int) < 0) + if (tracemalloc_start(nframe_int) < 0) { +DEBUG("start(): ERROR!"); return NULL; + } +DEBUG("start(): done"); Py_RETURN_NONE; } -- cgit v0.12 From ff9c5346ea4388bb0f77c9f92415fbfbc92e429f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 16:29:02 +0100 Subject: Issue #26588: fix compilation on Windows --- Include/pymem.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Include/pymem.h b/Include/pymem.h index 941e00f..3fac523 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -59,6 +59,8 @@ PyAPI_FUNC(int) _PyTraceMalloc_Untrack( PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr); + +PyAPI_DATA(int) tracemalloc_debug; #endif /* !Py_LIMITED_API */ -- cgit v0.12 From 0cfc058d619cea398baf34bfdd94b52620275f88 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 17:40:07 +0100 Subject: Issue #26588: more assertions --- Modules/_tracemalloc.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 551bade..baeb58c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -189,11 +189,11 @@ set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); if (reentrant) { - assert(PyThread_get_key_value(tracemalloc_reentrant_key) == NULL); + assert(!get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); } else { - assert(PyThread_get_key_value(tracemalloc_reentrant_key) == REENTRANT); + assert(get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, NULL); } } @@ -901,6 +901,11 @@ static int tracemalloc_init(void) { DEBUG("tracemalloc_init()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); @@ -1027,6 +1032,11 @@ tracemalloc_start(int max_nframe) size_t size; DEBUG("tracemalloc_start()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (tracemalloc_init() < 0) { DEBUG("tracemalloc_start(): ERROR! init failed!"); return -1; @@ -1035,8 +1045,10 @@ DEBUG("tracemalloc_start(): ERROR! init failed!"); if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ DEBUG("tracemalloc_start(): exit (already tracing)"); +assert(!get_reentrant()); return 0; } +assert(get_reentrant()); assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); tracemalloc_config.max_nframe = max_nframe; @@ -1081,6 +1093,7 @@ DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); DEBUG("tracemalloc_start(): done"); +assert(!get_reentrant()); return 0; } @@ -1089,10 +1102,17 @@ static void tracemalloc_stop(void) { DEBUG("tracemalloc_stop()"); + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + if (!tracemalloc_config.tracing) { DEBUG("tracemalloc_stop(): exit (not tracing)"); +assert(get_reentrant()); return; } +assert(!get_reentrant()); /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -1115,6 +1135,7 @@ DEBUG("tracemalloc_stop(): set_reentrant(1)"); raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; DEBUG("tracemalloc_stop(): done"); +assert(get_reentrant()); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, -- cgit v0.12 From 4a06647534162c7a1bed0ec01d544758b56a446b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 17:45:09 +0100 Subject: Add assertions on tracemalloc_reentrant_key Issue #26588. --- Modules/_tracemalloc.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index baeb58c..5c9f69e 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -167,7 +167,7 @@ tracemalloc_error(const char *format, ...) # error "need native thread local storage (TLS)" #endif -static int tracemalloc_reentrant_key; +static int tracemalloc_reentrant_key = -1; /* Any non-NULL pointer can be used */ #define REENTRANT Py_True @@ -175,7 +175,10 @@ static int tracemalloc_reentrant_key; static int get_reentrant(void) { - void *ptr = PyThread_get_key_value(tracemalloc_reentrant_key); + void *ptr; + + assert(tracemalloc_reentrant_key != -1); + ptr = PyThread_get_key_value(tracemalloc_reentrant_key); if (ptr != NULL) { assert(ptr == REENTRANT); return 1; @@ -188,6 +191,8 @@ static void set_reentrant(int reentrant) { assert(reentrant == 0 || reentrant == 1); + assert(tracemalloc_reentrant_key != -1); + if (reentrant) { assert(!get_reentrant()); PyThread_set_key_value(tracemalloc_reentrant_key, REENTRANT); @@ -1018,6 +1023,7 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL PyThread_delete_key(tracemalloc_reentrant_key); + tracemalloc_reentrant_key = -1; #endif Py_XDECREF(unknown_filename); -- cgit v0.12 From e31e35d34e9db28d36481c9921b3ffdcc65d750d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 18:48:50 +0100 Subject: Issue #26588: one more assertion --- Modules/_tracemalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 5c9f69e..288942a 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -992,6 +992,7 @@ DEBUG("tracemalloc_init(): set_reentrant(1)"); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; DEBUG("tracemalloc_init(): done"); +assert(get_reentrant()); return 0; } -- cgit v0.12 From b8d927266d6c1f098b3490b897e160222d035e5c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 20:56:49 +0100 Subject: Issue #26588: Don't call tracemalloc_init() at module initilization So it's possible to get debug messages in test_tracemalloc. --- Modules/_tracemalloc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 288942a..79a7164 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1644,8 +1644,10 @@ PyInit__tracemalloc(void) if (m == NULL) return NULL; +#if 0 if (tracemalloc_init() < 0) return NULL; +#endif return m; } -- cgit v0.12 From eddb074c534e32caad468c952c7d00adcbd76a41 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 21:06:07 +0100 Subject: Issue #26588: more debug traces --- Modules/_tracemalloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 79a7164..a674d0c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -899,7 +899,7 @@ tracemalloc_clear_traces(void) } #define DEBUG(MSG) \ - if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li] " MSG "\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); } + if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li, reentrant key %i] " MSG "\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); } static int @@ -926,6 +926,7 @@ DEBUG("tracemalloc_init(): exit (already initialized)"); #ifdef REENTRANT_THREADLOCAL tracemalloc_reentrant_key = PyThread_create_key(); +fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); if (tracemalloc_reentrant_key == -1) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); @@ -1023,6 +1024,7 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL +fprintf(stderr, "[pid %li, tid %li] PyThread_delete_key(%i)\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); PyThread_delete_key(tracemalloc_reentrant_key); tracemalloc_reentrant_key = -1; #endif @@ -1640,6 +1642,9 @@ PyMODINIT_FUNC PyInit__tracemalloc(void) { PyObject *m; + +fprintf(stderr, "[pid %li, tid %li] PyInit__tracemalloc\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); + m = PyModule_Create(&module_def); if (m == NULL) return NULL; -- cgit v0.12 From 92c21d7a7c3a925358e606ffa7ab25cc7e73f0b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 21:26:31 +0100 Subject: Issue #26588: skip test_warnings.test_tracemalloc() --- Lib/test/test_warnings/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e6f47cd..f2401c8 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -772,6 +772,7 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings + @unittest.skipIf(True, "FIXME: Issue #26588") def test_tracemalloc(self): self.addCleanup(support.unlink, support.TESTFN) -- cgit v0.12 From f9a71153e9122a3d2bb86fc479bdda2f2adc859e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2016 23:54:42 +0100 Subject: Issue #26588: remove debug traces from _tracemalloc. --- Include/pymem.h | 2 -- Lib/test/test_tracemalloc.py | 10 ------ Lib/test/test_warnings/__init__.py | 1 - Modules/_testcapimodule.c | 14 -------- Modules/_tracemalloc.c | 69 +++----------------------------------- 5 files changed, 5 insertions(+), 91 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index 3fac523..941e00f 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -59,8 +59,6 @@ PyAPI_FUNC(int) _PyTraceMalloc_Untrack( PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr); - -PyAPI_DATA(int) tracemalloc_debug; #endif /* !Py_LIMITED_API */ diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 44cd539..359d9c0 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -88,9 +88,6 @@ def traceback_filename(filename): class TestTracemallocEnabled(unittest.TestCase): def setUp(self): - if _testcapi: - _testcapi.tracemalloc_set_debug(True) - if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -98,8 +95,6 @@ class TestTracemallocEnabled(unittest.TestCase): def tearDown(self): tracemalloc.stop() - if _testcapi: - _testcapi.tracemalloc_set_debug(False) def test_get_tracemalloc_memory(self): data = [allocate_bytes(123) for count in range(1000)] @@ -882,9 +877,6 @@ class TestCAPI(unittest.TestCase): maxDiff = 80 * 20 def setUp(self): - if _testcapi: - _testcapi.tracemalloc_set_debug(True) - if tracemalloc.is_tracing(): self.skipTest("tracemalloc must be stopped before the test") @@ -898,8 +890,6 @@ class TestCAPI(unittest.TestCase): def tearDown(self): tracemalloc.stop() - if _testcapi: - _testcapi.tracemalloc_set_debug(False) def get_traceback(self): frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index f2401c8..e6f47cd 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -772,7 +772,6 @@ class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = py_warnings - @unittest.skipIf(True, "FIXME: Issue #26588") def test_tracemalloc(self): self.addCleanup(support.unlink, support.TESTFN) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f952aec..8c79485 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3747,19 +3747,6 @@ tracemalloc_get_traceback(PyObject *self, PyObject *args) return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr); } -PyObject* -tracemalloc_set_debug(PyObject *self, PyObject *args) -{ - int debug; - extern int tracemalloc_debug; - - if (!PyArg_ParseTuple(args, "i", &debug)) - return NULL; - - tracemalloc_debug = debug; - Py_RETURN_NONE; -} - static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3949,7 +3936,6 @@ static PyMethodDef TestMethods[] = { {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, - {"tracemalloc_set_debug", tracemalloc_set_debug, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index a674d0c..77742de 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -45,8 +45,6 @@ static struct { int use_domain; } tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; -int tracemalloc_debug = 0; - #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without the GIL held from PyMem_RawFree(). It cannot acquire the lock because it @@ -898,35 +896,23 @@ tracemalloc_clear_traces(void) _Py_hashtable_clear(tracemalloc_filenames); } -#define DEBUG(MSG) \ - if (tracemalloc_debug) { fprintf(stderr, "[pid %li, tid %li, reentrant key %i] " MSG "\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); } - static int tracemalloc_init(void) { -DEBUG("tracemalloc_init()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - if (tracemalloc_config.initialized == TRACEMALLOC_FINALIZED) { PyErr_SetString(PyExc_RuntimeError, "the tracemalloc module has been unloaded"); return -1; } - if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) { -DEBUG("tracemalloc_init(): exit (already initialized)"); + if (tracemalloc_config.initialized == TRACEMALLOC_INITIALIZED) return 0; - } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); #ifdef REENTRANT_THREADLOCAL tracemalloc_reentrant_key = PyThread_create_key(); -fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); if (tracemalloc_reentrant_key == -1) { #ifdef MS_WINDOWS PyErr_SetFromWindowsErr(0); @@ -988,12 +974,9 @@ fprintf(stderr, "[pid %li, tid %li] PyThread_create_key() -> %i\n", (long)getpid /* Disable tracing allocations until hooks are installed. Set also the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ -DEBUG("tracemalloc_init(): set_reentrant(1)"); set_reentrant(1); tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED; -DEBUG("tracemalloc_init(): done"); -assert(get_reentrant()); return 0; } @@ -1001,11 +984,8 @@ assert(get_reentrant()); static void tracemalloc_deinit(void) { -DEBUG("tracemalloc_deinit()"); - if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) { -DEBUG("tracemalloc_deinit(): exit (not initialized)"); + if (tracemalloc_config.initialized != TRACEMALLOC_INITIALIZED) return; - } tracemalloc_config.initialized = TRACEMALLOC_FINALIZED; tracemalloc_stop(); @@ -1022,15 +1002,12 @@ DEBUG("tracemalloc_deinit(): exit (not initialized)"); } #endif -DEBUG("tracemalloc_deinit(): delete reentrant key"); #ifdef REENTRANT_THREADLOCAL -fprintf(stderr, "[pid %li, tid %li] PyThread_delete_key(%i)\n", (long)getpid(), PyThread_get_thread_ident(), tracemalloc_reentrant_key); fflush(stderr); PyThread_delete_key(tracemalloc_reentrant_key); tracemalloc_reentrant_key = -1; #endif Py_XDECREF(unknown_filename); -DEBUG("tracemalloc_deinit(): done"); } @@ -1040,24 +1017,13 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; -DEBUG("tracemalloc_start()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - - if (tracemalloc_init() < 0) { -DEBUG("tracemalloc_start(): ERROR! init failed!"); + if (tracemalloc_init() < 0) return -1; - } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ -DEBUG("tracemalloc_start(): exit (already tracing)"); -assert(!get_reentrant()); return 0; } -assert(get_reentrant()); assert(1 <= max_nframe && max_nframe <= MAX_NFRAME); tracemalloc_config.max_nframe = max_nframe; @@ -1097,12 +1063,8 @@ assert(get_reentrant()); /* everything is ready: start tracing Python memory allocations */ tracemalloc_config.tracing = 1; - -DEBUG("tracemalloc_start(): set_reentrant(0)"); set_reentrant(0); -DEBUG("tracemalloc_start(): done"); -assert(!get_reentrant()); return 0; } @@ -1110,25 +1072,14 @@ assert(!get_reentrant()); static void tracemalloc_stop(void) { -DEBUG("tracemalloc_stop()"); - -#ifdef WITH_THREAD - assert(PyGILState_Check()); -#endif - - if (!tracemalloc_config.tracing) { -DEBUG("tracemalloc_stop(): exit (not tracing)"); -assert(get_reentrant()); + if (!tracemalloc_config.tracing) return; - } -assert(!get_reentrant()); /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; /* set the reentrant flag to detect bugs: fail with an assertion error if set_reentrant(1) is called while tracing is disabled. */ -DEBUG("tracemalloc_stop(): set_reentrant(1)"); set_reentrant(1); /* unregister the hook on memory allocators */ @@ -1143,8 +1094,6 @@ DEBUG("tracemalloc_stop(): set_reentrant(1)"); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; -DEBUG("tracemalloc_stop(): done"); -assert(get_reentrant()); } PyDoc_STRVAR(tracemalloc_is_tracing_doc, @@ -1512,11 +1461,8 @@ py_tracemalloc_start(PyObject *self, PyObject *args) } nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - if (tracemalloc_start(nframe_int) < 0) { -DEBUG("start(): ERROR!"); + if (tracemalloc_start(nframe_int) < 0) return NULL; - } -DEBUG("start(): done"); Py_RETURN_NONE; } @@ -1642,17 +1588,12 @@ PyMODINIT_FUNC PyInit__tracemalloc(void) { PyObject *m; - -fprintf(stderr, "[pid %li, tid %li] PyInit__tracemalloc\n", (long)getpid(), PyThread_get_thread_ident()); fflush(stderr); - m = PyModule_Create(&module_def); if (m == NULL) return NULL; -#if 0 if (tracemalloc_init() < 0) return NULL; -#endif return m; } -- cgit v0.12 From 060f9bb6024580272440f57c612ca34d9beaa169 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:18:36 +0100 Subject: Fix macros in hashtable.h Add parenthesis. --- Modules/hashtable.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 41542d2..4199aab 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -146,10 +146,10 @@ PyAPI_FUNC(int) _Py_hashtable_set( const void *data); #define _Py_HASHTABLE_SET(TABLE, KEY, DATA) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, sizeof(DATA), &(DATA)) + _Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), sizeof(DATA), &(DATA)) #define _Py_HASHTABLE_SET_NODATA(TABLE, KEY) \ - _Py_hashtable_set(TABLE, sizeof(KEY), &KEY, 0, NULL) + _Py_hashtable_set(TABLE, sizeof(KEY), &(KEY), 0, NULL) /* Get an entry. -- cgit v0.12 From e19558af1b6979edf27f7a05a6e47a8b5d113390 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:28:08 +0100 Subject: Add a source parameter to warnings.warn() Issue #26604: * Add a new optional source parameter to _warnings.warn() and warnings.warn() * Modify asyncore, asyncio and _pyio modules to set the source parameter when logging a ResourceWarning warning --- Doc/library/warnings.rst | 8 +++++++- Lib/_pyio.py | 2 +- Lib/asyncio/base_events.py | 3 ++- Lib/asyncio/base_subprocess.py | 3 ++- Lib/asyncio/proactor_events.py | 3 ++- Lib/asyncio/selector_events.py | 3 ++- Lib/asyncio/sslproto.py | 3 ++- Lib/asyncio/unix_events.py | 6 ++++-- Lib/asyncio/windows_utils.py | 3 ++- Lib/asyncore.py | 3 ++- Lib/tempfile.py | 1 - Lib/warnings.py | 4 ++-- Python/_warnings.c | 11 ++++++----- 13 files changed, 34 insertions(+), 19 deletions(-) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 4ce88ab..4fec365 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -300,7 +300,7 @@ Available Functions ------------------- -.. function:: warn(message, category=None, stacklevel=1) +.. function:: warn(message, category=None, stacklevel=1, source=None) Issue a warning, or maybe ignore it or raise an exception. The *category* argument, if given, must be a warning category class (see above); it defaults to @@ -318,6 +318,12 @@ Available Functions source of :func:`deprecation` itself (since the latter would defeat the purpose of the warning message). + *source*, if supplied, is the destroyed object which emitted a + :exc:`ResourceWarning`. + + .. versionchanged:: 3.6 + Added *source* parameter. + .. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index c3ad81e..972c082 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1514,7 +1514,7 @@ class FileIO(RawIOBase): if self._fd >= 0 and self._closefd and not self.closed: import warnings warnings.warn('unclosed file %r' % (self,), ResourceWarning, - stacklevel=2) + stacklevel=2, source=self) self.close() def __getstate__(self): diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 9d07673..3a42b10 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -412,7 +412,8 @@ class BaseEventLoop(events.AbstractEventLoop): if compat.PY34: def __del__(self): if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning) + warnings.warn("unclosed event loop %r" % self, ResourceWarning, + source=self) if not self.is_running(): self.close() diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 73425d9..efe0831 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -122,7 +122,8 @@ class BaseSubprocessTransport(transports.SubprocessTransport): if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def get_pid(self): diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 14c0659..2671a94 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -86,7 +86,8 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 812fac1..cbb3625 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -573,7 +573,8 @@ class _SelectorTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._sock.close() def _fatal_error(self, exc, message='Fatal error on transport'): diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index dde980b..1cea850 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -324,7 +324,8 @@ class _SSLProtocolTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self.close() def pause_reading(self): diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 7747ff4..2beba3e 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -378,7 +378,8 @@ class _UnixReadPipeTransport(transports.ReadTransport): if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): @@ -567,7 +568,8 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, if compat.PY34: def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning) + warnings.warn("unclosed transport %r" % self, ResourceWarning, + source=self) self._pipe.close() def abort(self): diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index 870cd13..7c63fb9 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -159,7 +159,8 @@ class PipeHandle: def __del__(self): if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning) + warnings.warn("unclosed %r" % self, ResourceWarning, + source=self) self.close() def __enter__(self): diff --git a/Lib/asyncore.py b/Lib/asyncore.py index 3b51f0f..4b046d6 100644 --- a/Lib/asyncore.py +++ b/Lib/asyncore.py @@ -595,7 +595,8 @@ if os.name == 'posix': def __del__(self): if self.fd >= 0: - warnings.warn("unclosed file %r" % self, ResourceWarning) + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) self.close() def recv(self, *args): diff --git a/Lib/tempfile.py b/Lib/tempfile.py index ad687b9..6146235 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -797,7 +797,6 @@ class TemporaryDirectory(object): _shutil.rmtree(name) _warnings.warn(warn_message, ResourceWarning) - def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.name) diff --git a/Lib/warnings.py b/Lib/warnings.py index 9fb21a8..d4f591e 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -233,7 +233,7 @@ def _next_external_frame(frame): # Code typically replaced by _warnings -def warn(message, category=None, stacklevel=1): +def warn(message, category=None, stacklevel=1, source=None): """Issue a warning, or maybe ignore it or raise an exception.""" # Check if message is already a Warning object if isinstance(message, Warning): @@ -283,7 +283,7 @@ def warn(message, category=None, stacklevel=1): filename = module registry = globals.setdefault("__warningregistry__", {}) warn_explicit(message, category, filename, lineno, module, registry, - globals) + globals, source) def warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, diff --git a/Python/_warnings.c b/Python/_warnings.c index 25299fb..dcac57b 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -787,18 +787,19 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level, static PyObject * warnings_warn(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kw_list[] = { "message", "category", "stacklevel", 0 }; - PyObject *message, *category = NULL; + static char *kw_list[] = {"message", "category", "stacklevel", + "source", NULL}; + PyObject *message, *category = NULL, *source = NULL; Py_ssize_t stack_level = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|On:warn", kw_list, - &message, &category, &stack_level)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OnO:warn", kw_list, + &message, &category, &stack_level, &source)) return NULL; category = get_category(message, category); if (category == NULL) return NULL; - return do_warn(message, category, stack_level, NULL); + return do_warn(message, category, stack_level, source); } static PyObject * -- cgit v0.12 From 7bfa409ff841fd84dfa194dd9052650d0a28585d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:43:54 +0100 Subject: Implement finalizer for os.scandir() iterator Issue #26603: * Implement finalizer for os.scandir() iterator * Set the source parameter when emitting the ResourceWarning warning * Close the iterator before emitting the warning --- Modules/posixmodule.c | 64 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65b20be..1cd0f24 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -12101,29 +12101,38 @@ ScandirIterator_exit(ScandirIterator *self, PyObject *args) } static void -ScandirIterator_dealloc(ScandirIterator *iterator) +ScandirIterator_finalize(ScandirIterator *iterator) { + PyObject *error_type, *error_value, *error_traceback; + + /* Save the current exception, if any. */ + PyErr_Fetch(&error_type, &error_value, &error_traceback); + if (!ScandirIterator_is_closed(iterator)) { - PyObject *exc, *val, *tb; - Py_ssize_t old_refcount = Py_REFCNT(iterator); - /* Py_INCREF/Py_DECREF cannot be used, because the refcount is - * likely zero, Py_DECREF would call again the destructor. - */ - ++Py_REFCNT(iterator); - PyErr_Fetch(&exc, &val, &tb); - if (PyErr_WarnFormat(PyExc_ResourceWarning, 1, - "unclosed scandir iterator %R", iterator)) { + ScandirIterator_closedir(iterator); + + if (PyErr_ResourceWarning((PyObject *)iterator, 1, + "unclosed scandir iterator %R", iterator)) { /* Spurious errors can appear at shutdown */ - if (PyErr_ExceptionMatches(PyExc_Warning)) + if (PyErr_ExceptionMatches(PyExc_Warning)) { PyErr_WriteUnraisable((PyObject *) iterator); + } } - PyErr_Restore(exc, val, tb); - Py_REFCNT(iterator) = old_refcount; - - ScandirIterator_closedir(iterator); } - Py_XDECREF(iterator->path.object); + + Py_CLEAR(iterator->path.object); path_cleanup(&iterator->path); + + /* Restore the saved exception. */ + PyErr_Restore(error_type, error_value, error_traceback); +} + +static void +ScandirIterator_dealloc(ScandirIterator *iterator) +{ + if (PyObject_CallFinalizerFromDealloc((PyObject *)iterator) < 0) + return; + Py_TYPE(iterator)->tp_free((PyObject *)iterator); } @@ -12155,7 +12164,8 @@ static PyTypeObject ScandirIteratorType = { 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ 0, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ @@ -12164,6 +12174,26 @@ static PyTypeObject ScandirIteratorType = { PyObject_SelfIter, /* tp_iter */ (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ ScandirIterator_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ + (destructor)ScandirIterator_finalize, /* tp_finalize */ }; static PyObject * -- cgit v0.12 From e98445a4deb2b2eb97de26e03fc8c4c2a5f256d4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 00:54:48 +0100 Subject: _warnings.warn_explicit(): try to import warnings Issue #26592: _warnings.warn_explicit() now tries to import the warnings module (Python implementation) if the source parameter is set to be able to log the traceback where the source was allocated. --- Python/_warnings.c | 50 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index dcac57b..41eaf53 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -40,11 +40,11 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr) +get_warnings_attr(const char *attr, int try_import) { static PyObject *warnings_str = NULL; PyObject *all_modules; - PyObject *warnings_module; + PyObject *warnings_module, *obj; int result; if (warnings_str == NULL) { @@ -53,15 +53,34 @@ get_warnings_attr(const char *attr) return NULL; } - all_modules = PyImport_GetModuleDict(); - result = PyDict_Contains(all_modules, warnings_str); - if (result == -1 || result == 0) + /* don't try to import after the start of the Python finallization */ + if (try_import && _Py_Finalizing == NULL) { + warnings_module = PyImport_Import(warnings_str); + if (warnings_module == NULL) { + /* Fallback to the C implementation if we cannot get + the Python implementation */ + PyErr_Clear(); + return NULL; + } + } + else { + all_modules = PyImport_GetModuleDict(); + result = PyDict_Contains(all_modules, warnings_str); + if (result == -1 || result == 0) + return NULL; + + warnings_module = PyDict_GetItem(all_modules, warnings_str); + Py_INCREF(warnings_module); + } + + if (!PyObject_HasAttrString(warnings_module, attr)) { + Py_DECREF(warnings_module); return NULL; + } - warnings_module = PyDict_GetItem(all_modules, warnings_str); - if (!PyObject_HasAttrString(warnings_module, attr)) - return NULL; - return PyObject_GetAttrString(warnings_module, attr); + obj = PyObject_GetAttrString(warnings_module, attr); + Py_DECREF(warnings_module); + return obj; } @@ -70,7 +89,7 @@ get_once_registry(void) { PyObject *registry; - registry = get_warnings_attr("onceregistry"); + registry = get_warnings_attr("onceregistry", 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; @@ -87,7 +106,7 @@ get_default_action(void) { PyObject *default_action; - default_action = get_warnings_attr("defaultaction"); + default_action = get_warnings_attr("defaultaction", 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; @@ -110,7 +129,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_ssize_t i; PyObject *warnings_filters; - warnings_filters = get_warnings_attr("filters"); + warnings_filters = get_warnings_attr("filters", 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; @@ -366,7 +385,10 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; - show_fn = get_warnings_attr("_showwarnmsg"); + /* If the source parameter is set, try to get the Python implementation. + The Python implementation is able to log the traceback where the source + was allocated, whereas the C implementation doesnt. */ + show_fn = get_warnings_attr("_showwarnmsg", source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -380,7 +402,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage"); + warnmsg_cls = get_warnings_attr("WarningMessage", 0); if (warnmsg_cls == NULL) { PyErr_SetString(PyExc_RuntimeError, "unable to get warnings.WarningMessage"); -- cgit v0.12 From 6d7f4f6675b683475a8eaa8f0f5dff35a0b165f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 02:04:32 +0100 Subject: regrtest: add timeout to main process when using -jN libregrtest: add a watchdog to run_tests_multiprocess() using faulthandler.dump_traceback_later(). --- Lib/test/libregrtest/runtest_mp.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 0ca7dd7..5e847a0 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -1,3 +1,4 @@ +import faulthandler import json import os import queue @@ -151,6 +152,8 @@ class MultiprocessThread(threading.Thread): def run_tests_multiprocess(regrtest): output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) + test_timeout = regrtest.ns.timeout + use_timeout = (test_timeout is not None) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] @@ -170,11 +173,14 @@ def run_tests_multiprocess(regrtest): finished = 0 test_index = 1 - timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) + get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: + if use_timeout: + faulthandler.dump_traceback_later(test_timeout, exit=True) + try: - item = output.get(timeout=timeout) + item = output.get(timeout=get_timeout) except queue.Empty: running = get_running(workers) if running and not regrtest.ns.pgo: @@ -215,6 +221,9 @@ def run_tests_multiprocess(regrtest): regrtest.interrupted = True pending.interrupted = True print() + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() running = [worker.current_test for worker in workers] running = list(filter(bool, running)) -- cgit v0.12 From d65e0c7560f8a5ad31d31892cee6f1416293ca39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 02:05:39 +0100 Subject: Makefile: change default value of TESTTIMEOUT from 1 hour to 15 min The whole test suite takes 6 minutes on my laptop. It takes less than 30 minutes on most buildbots. The TESTTIMEOUT is the timeout for a single test file. --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6514bf8..5c90daf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -974,7 +974,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) TESTOPTS= $(EXTRATESTOPTS) TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py -TESTTIMEOUT= 3600 +TESTTIMEOUT= 900 # Run a basic set of regression tests. # This excludes some tests that are particularly resource-intensive. -- cgit v0.12 From 42bcf37fcffa65592e401c43aa8c0190452b28b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:08:08 +0100 Subject: Issue #26588: Optimize tracemalloc_realloc() No need to remove the old trace if the memory block didn't move. --- Modules/_tracemalloc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 3d96860..139b169 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -633,7 +633,12 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) /* an existing memory block has been resized */ TABLES_LOCK(); - REMOVE_TRACE(ptr); + + /* tracemalloc_add_trace() updates the trace if there is already + a trace at address (domain, ptr2) */ + if (ptr2 != ptr) { + REMOVE_TRACE(ptr); + } if (ADD_TRACE(ptr2, new_size) < 0) { /* Memory allocation failed. The error cannot be reported to -- cgit v0.12 From e8c6b2fd1bb75c6a3865745de30f26e235d3f12f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:25:01 +0100 Subject: Issue #26588: * _Py_HASHTABLE_ENTRY_DATA: change type from "char *" to "const void *" * Add _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro * Rename _Py_HASHTABLE_ENTRY_WRITE_DATA() macro to _Py_HASHTABLE_ENTRY_WRITE_PDATA() * Add _Py_HASHTABLE_ENTRY_WRITE_DATA() macro --- Modules/_tracemalloc.c | 2 +- Modules/hashtable.c | 10 ++++------ Modules/hashtable.h | 27 +++++++++++++++++++++++---- Python/marshal.c | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 139b169..0bab540 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1263,7 +1263,7 @@ tracemalloc_pyobject_decref_cb(_Py_hashtable_t *tracebacks, void *user_data) { PyObject *obj; - _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, sizeof(obj), &obj); + _Py_HASHTABLE_ENTRY_READ_DATA(tracebacks, entry, obj); Py_DECREF(obj); return 0; } diff --git a/Modules/hashtable.c b/Modules/hashtable.c index bb20cce..d80acc6 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -287,7 +287,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); + _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -325,10 +325,8 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } entry->key_hash = key_hash; - memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(entry), pkey, key_size); - - assert(data_size == ht->data_size); - memcpy(_Py_HASHTABLE_ENTRY_DATA(ht, entry), data, data_size); + _Py_HASHTABLE_ENTRY_WRITE_PKEY(key_size, entry, pkey); + _Py_HASHTABLE_ENTRY_WRITE_PDATA(ht, entry, data_size, data); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -350,7 +348,7 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_DATA(ht, entry, data_size, data); + _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); return 1; } diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 4199aab..9c3fbdd 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -30,10 +30,13 @@ typedef struct { } _Py_hashtable_entry_t; #define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ - ((const void *)((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t))) + ((const void *)((char *)(ENTRY) \ + + sizeof(_Py_hashtable_entry_t))) #define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ - ((char *)(ENTRY) + sizeof(_Py_hashtable_entry_t) + (TABLE)->key_size) + ((const void *)((char *)(ENTRY) \ + + sizeof(_Py_hashtable_entry_t) \ + + (TABLE)->key_size)) /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ @@ -49,10 +52,26 @@ typedef struct { memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ } while (0) -#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA_SIZE, DATA) \ +#define _Py_HASHTABLE_ENTRY_WRITE_PKEY(KEY_SIZE, ENTRY, PKEY) \ + do { \ + memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(ENTRY), (PKEY), (KEY_SIZE)); \ + } while (0) + +#define _Py_HASHTABLE_ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy((PDATA), _Py_HASHTABLE_ENTRY_DATA(TABLE, (ENTRY)), \ + (DATA_SIZE)); \ + } while (0) + +#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \ + _Py_HASHTABLE_ENTRY_READ_PDATA((TABLE), (ENTRY), sizeof(DATA), &(DATA)) + +#define _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ do { \ assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy(DATA, _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY), DATA_SIZE); \ + memcpy((void *)_Py_HASHTABLE_ENTRY_DATA((TABLE), (ENTRY)), \ + (PDATA), (DATA_SIZE)); \ } while (0) diff --git a/Python/marshal.c b/Python/marshal.c index 83a1885..3be77a8 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -266,7 +266,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_HASHTABLE_GET_ENTRY(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, sizeof(w), &w); + _Py_HASHTABLE_ENTRY_READ_DATA(p->hashtable, entry, w); /* we don't store "long" indices in the dict */ assert(0 <= w && w <= 0x7fffffff); w_byte(TYPE_REF, p); -- cgit v0.12 From ca79ccd9e69641330d4002acac1bfeeb2dccda32 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:38:54 +0100 Subject: Issue #26588: * Optimize tracemalloc_add_trace(): modify hashtable entry data (trace) if the memory block is already tracked, rather than trying to remove the old trace and then add a new trace. * Add _Py_HASHTABLE_ENTRY_WRITE_DATA() macro --- Include/pymem.h | 2 +- Modules/_tracemalloc.c | 38 +++++++++++++++++++++++++++----------- Modules/hashtable.h | 3 +++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/Include/pymem.h b/Include/pymem.h index 941e00f..431e5b6 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -34,7 +34,7 @@ typedef unsigned int _PyTraceMalloc_domain_t; Return -2 if tracemalloc is disabled. - If memory block was already tracked, begin by removing the old trace. */ + If memory block is already tracked, update the existing trace. */ PyAPI_FUNC(int) _PyTraceMalloc_Track( _PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 0bab540..e6465a3 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -552,33 +552,49 @@ static int tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, size_t size) { + pointer_t key = {ptr, domain}; traceback_t *traceback; trace_t trace; + _Py_hashtable_entry_t* entry; int res; assert(tracemalloc_config.tracing); - /* first, remove the previous trace (if any) */ - tracemalloc_remove_trace(domain, ptr); - traceback = traceback_new(); if (traceback == NULL) { return -1; } - trace.size = size; - trace.traceback = traceback; - if (tracemalloc_config.use_domain) { - pointer_t key = {ptr, domain}; - res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, key); } else { - res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, ptr); } - if (res != 0) { - return res; + if (entry != NULL) { + /* the memory block is already tracked */ + _Py_HASHTABLE_ENTRY_READ_DATA(tracemalloc_traces, entry, trace); + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; + + trace.size = size; + trace.traceback = traceback; + _Py_HASHTABLE_ENTRY_WRITE_DATA(tracemalloc_traces, entry, trace); + } + else { + trace.size = size; + trace.traceback = traceback; + + if (tracemalloc_config.use_domain) { + res = _Py_HASHTABLE_SET(tracemalloc_traces, key, trace); + } + else { + res = _Py_HASHTABLE_SET(tracemalloc_traces, ptr, trace); + } + if (res != 0) { + return res; + } } assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); diff --git a/Modules/hashtable.h b/Modules/hashtable.h index 9c3fbdd..e3e8148 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -74,6 +74,9 @@ typedef struct { (PDATA), (DATA_SIZE)); \ } while (0) +#define _Py_HASHTABLE_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA) \ + _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, sizeof(DATA), &(DATA)) + /* _Py_hashtable: prototypes */ -- cgit v0.12 From 5dacbd4c42171e447e2f07144faf502774dc921a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 09:52:13 +0100 Subject: Cleanup hashtable.h Issue #26588: * Pass the hash table rather than the key size to hash and compare functions * _Py_HASHTABLE_READ_KEY() and _Py_HASHTABLE_ENTRY_READ_KEY() macros now expect the hash table as the first parameter, rather than the key size * tracemalloc_get_traces_fill(): use _Py_HASHTABLE_ENTRY_READ_DATA() rather than pointer dereference * Remove the _Py_HASHTABLE_ENTRY_WRITE_PKEY() macro * Move "PKEY" and "PDATA" macros inside hashtable.c --- Modules/_tracemalloc.c | 50 ++++++++++++++++++++++---------------------- Modules/hashtable.c | 55 +++++++++++++++++++++++++++++-------------------- Modules/hashtable.h | 56 ++++++++++++++++++++++---------------------------- Python/marshal.c | 2 +- 4 files changed, 82 insertions(+), 81 deletions(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index e6465a3..169fd2c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -223,23 +223,23 @@ set_reentrant(int reentrant) static Py_uhash_t -hashtable_hash_pyobject(size_t key_size, const void *pkey) +hashtable_hash_pyobject(_Py_hashtable_t *ht, const void *pkey) { PyObject *obj; - _Py_HASHTABLE_READ_KEY(key_size, pkey, obj); + _Py_HASHTABLE_READ_KEY(ht, pkey, obj); return PyObject_Hash(obj); } static int -hashtable_compare_unicode(size_t key_size, const void *pkey, +hashtable_compare_unicode(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { PyObject *key1, *key2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, key2); + _Py_HASHTABLE_READ_KEY(ht, pkey, key1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, key2); if (key1 != NULL && key2 != NULL) return (PyUnicode_Compare(key1, key2) == 0); @@ -249,12 +249,12 @@ hashtable_compare_unicode(size_t key_size, const void *pkey, static Py_uhash_t -hashtable_hash_pointer_t(size_t key_size, const void *pkey) +hashtable_hash_pointer_t(_Py_hashtable_t *ht, const void *pkey) { pointer_t ptr; Py_uhash_t hash; - _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr); + _Py_HASHTABLE_READ_KEY(ht, pkey, ptr); hash = (Py_uhash_t)_Py_HashPointer((void*)ptr.ptr); hash ^= ptr.domain; @@ -263,13 +263,13 @@ hashtable_hash_pointer_t(size_t key_size, const void *pkey) int -hashtable_compare_pointer_t(size_t key_size, const void *pkey, +hashtable_compare_pointer_t(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { pointer_t ptr1, ptr2; - _Py_HASHTABLE_READ_KEY(key_size, pkey, ptr1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, entry, ptr2); + _Py_HASHTABLE_READ_KEY(ht, pkey, ptr1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, ptr2); /* compare pointer before domain, because pointer is more likely to be different */ @@ -304,25 +304,25 @@ raw_free(void *ptr) static Py_uhash_t -hashtable_hash_traceback(size_t key_size, const void *pkey) +hashtable_hash_traceback(_Py_hashtable_t *ht, const void *pkey) { traceback_t *traceback; - _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback); + _Py_HASHTABLE_READ_KEY(ht, pkey, traceback); return traceback->hash; } static int -hashtable_compare_traceback(size_t key_size, const void *pkey, - const _Py_hashtable_entry_t *he) +hashtable_compare_traceback(_Py_hashtable_t *ht, const void *pkey, + const _Py_hashtable_entry_t *entry) { traceback_t *traceback1, *traceback2; const frame_t *frame1, *frame2; int i; - _Py_HASHTABLE_READ_KEY(key_size, pkey, traceback1); - _Py_HASHTABLE_ENTRY_READ_KEY(key_size, he, traceback2); + _Py_HASHTABLE_READ_KEY(ht, pkey, traceback1); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback2); if (traceback1->nframe != traceback2->nframe) return 0; @@ -395,8 +395,7 @@ tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) /* intern the filename */ entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_filenames, filename); if (entry != NULL) { - _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames->key_size, entry, - filename); + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_filenames, entry, filename); } else { /* tracemalloc_filenames is responsible to keep a reference @@ -490,8 +489,7 @@ traceback_new(void) /* intern the traceback */ entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_tracebacks, traceback); if (entry != NULL) { - _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks->key_size, entry, - traceback); + _Py_HASHTABLE_ENTRY_READ_KEY(tracemalloc_tracebacks, entry, traceback); } else { traceback_t *copy; @@ -873,7 +871,7 @@ tracemalloc_clear_filename(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { PyObject *filename; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, filename); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, filename); Py_DECREF(filename); return 0; } @@ -885,7 +883,7 @@ traceback_free_traceback(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { traceback_t *traceback; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, traceback); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, traceback); raw_free(traceback); return 0; } @@ -1246,21 +1244,21 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, _Py_hashtable_entry_t *entr { get_traces_t *get_traces = user_data; _PyTraceMalloc_domain_t domain; - trace_t *trace; + trace_t trace; PyObject *tracemalloc_obj; int res; if (tracemalloc_config.use_domain) { pointer_t key; - _Py_HASHTABLE_ENTRY_READ_KEY(traces->key_size, entry, key); + _Py_HASHTABLE_ENTRY_READ_KEY(traces, entry, key); domain = key.domain; } else { domain = DEFAULT_DOMAIN; } - trace = (trace_t *)_Py_HASHTABLE_ENTRY_DATA(traces, entry); + _Py_HASHTABLE_ENTRY_READ_DATA(traces, entry, trace); - tracemalloc_obj = trace_to_pyobject(domain, trace, get_traces->tracebacks); + tracemalloc_obj = trace_to_pyobject(domain, &trace, get_traces->tracebacks); if (tracemalloc_obj == NULL) return 1; diff --git a/Modules/hashtable.c b/Modules/hashtable.c index d80acc6..b53cc24 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -61,6 +61,20 @@ #define HASHTABLE_ITEM_SIZE(HT) \ (sizeof(_Py_hashtable_entry_t) + (HT)->key_size + (HT)->data_size) +#define ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + Py_MEMCPY((PDATA), _Py_HASHTABLE_ENTRY_PDATA(TABLE, (ENTRY)), \ + (DATA_SIZE)); \ + } while (0) + +#define ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PDATA((TABLE), (ENTRY)), \ + (PDATA), (DATA_SIZE)); \ + } while (0) + /* Forward declaration */ static void hashtable_rehash(_Py_hashtable_t *ht); @@ -91,21 +105,21 @@ _Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, Py_uhash_t -_Py_hashtable_hash_ptr(size_t key_size, const void *pkey) +_Py_hashtable_hash_ptr(struct _Py_hashtable_t *ht, const void *pkey) { void *key; - _Py_HASHTABLE_READ_KEY(key_size, pkey, key); - return (Py_uhash_t)_Py_HashPointer((void *)key); + _Py_HASHTABLE_READ_KEY(ht, pkey, key); + return (Py_uhash_t)_Py_HashPointer(key); } int -_Py_hashtable_compare_direct(size_t key_size, const void *pkey, +_Py_hashtable_compare_direct(_Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry) { - const void *pkey2 = _Py_HASHTABLE_ENTRY_KEY(entry); - return (memcmp(pkey, pkey2, key_size) == 0); + const void *pkey2 = _Py_HASHTABLE_ENTRY_PKEY(entry); + return (memcmp(pkey, pkey2, ht->key_size) == 0); } @@ -245,12 +259,11 @@ _Py_hashtable_get_entry(_Py_hashtable_t *ht, assert(key_size == ht->key_size); - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry)) break; } @@ -268,13 +281,12 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, assert(key_size == ht->key_size); - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); previous = NULL; for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { - if (entry->key_hash == key_hash - && ht->compare_func(key_size, pkey, entry)) + if (entry->key_hash == key_hash && ht->compare_func(ht, pkey, entry)) break; previous = entry; } @@ -287,7 +299,7 @@ _Py_hashtable_pop_entry(_Py_hashtable_t *ht, size_t key_size, const void *pkey, ht->entries--; if (data != NULL) - _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); + ENTRY_READ_PDATA(ht, entry, data_size, data); ht->alloc.free(entry); if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) @@ -315,7 +327,7 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, assert(entry == NULL); #endif - key_hash = ht->hash_func(key_size, pkey); + key_hash = ht->hash_func(ht, pkey); index = key_hash & (ht->num_buckets - 1); entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); @@ -325,8 +337,8 @@ _Py_hashtable_set(_Py_hashtable_t *ht, size_t key_size, const void *pkey, } entry->key_hash = key_hash; - _Py_HASHTABLE_ENTRY_WRITE_PKEY(key_size, entry, pkey); - _Py_HASHTABLE_ENTRY_WRITE_PDATA(ht, entry, data_size, data); + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PKEY(entry), pkey, ht->key_size); + ENTRY_WRITE_PDATA(ht, entry, data_size, data); _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); ht->entries++; @@ -348,7 +360,7 @@ _Py_hashtable_get(_Py_hashtable_t *ht, size_t key_size,const void *pkey, entry = _Py_hashtable_get_entry(ht, key_size, pkey); if (entry == NULL) return 0; - _Py_HASHTABLE_ENTRY_READ_PDATA(ht, entry, data_size, data); + ENTRY_READ_PDATA(ht, entry, data_size, data); return 1; } @@ -399,7 +411,6 @@ _Py_hashtable_foreach(_Py_hashtable_t *ht, static void hashtable_rehash(_Py_hashtable_t *ht) { - const size_t key_size = ht->key_size; size_t buckets_size, new_size, bucket; _Py_slist_t *old_buckets = NULL; size_t old_num_buckets; @@ -429,7 +440,7 @@ hashtable_rehash(_Py_hashtable_t *ht) size_t entry_index; - assert(ht->hash_func(key_size, _Py_HASHTABLE_ENTRY_KEY(entry)) == entry->key_hash); + assert(ht->hash_func(ht, _Py_HASHTABLE_ENTRY_PKEY(entry)) == entry->key_hash); next = ENTRY_NEXT(entry); entry_index = entry->key_hash & (new_size - 1); @@ -499,9 +510,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src) for (bucket=0; bucket < src->num_buckets; bucket++) { entry = TABLE_HEAD(src, bucket); for (; entry; entry = ENTRY_NEXT(entry)) { - const void *pkey = _Py_HASHTABLE_ENTRY_KEY(entry); - const void *data = _Py_HASHTABLE_ENTRY_DATA(src, entry); - err = _Py_hashtable_set(dst, key_size, pkey, data_size, data); + const void *pkey = _Py_HASHTABLE_ENTRY_PKEY(entry); + const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(src, entry); + err = _Py_hashtable_set(dst, key_size, pkey, data_size, pdata); if (err) { _Py_hashtable_destroy(dst); return NULL; diff --git a/Modules/hashtable.h b/Modules/hashtable.h index e3e8148..18fed09 100644 --- a/Modules/hashtable.h +++ b/Modules/hashtable.h @@ -29,60 +29,52 @@ typedef struct { /* key (key_size bytes) and then data (data_size bytes) follows */ } _Py_hashtable_entry_t; -#define _Py_HASHTABLE_ENTRY_KEY(ENTRY) \ +#define _Py_HASHTABLE_ENTRY_PKEY(ENTRY) \ ((const void *)((char *)(ENTRY) \ + sizeof(_Py_hashtable_entry_t))) -#define _Py_HASHTABLE_ENTRY_DATA(TABLE, ENTRY) \ +#define _Py_HASHTABLE_ENTRY_PDATA(TABLE, ENTRY) \ ((const void *)((char *)(ENTRY) \ + sizeof(_Py_hashtable_entry_t) \ + (TABLE)->key_size)) /* Get a key value from pkey: use memcpy() rather than a pointer dereference to avoid memory alignment issues. */ -#define _Py_HASHTABLE_READ_KEY(KEY_SIZE, PKEY, DST_KEY) \ +#define _Py_HASHTABLE_READ_KEY(TABLE, PKEY, DST_KEY) \ do { \ - assert(sizeof(DST_KEY) == (KEY_SIZE)); \ - memcpy(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ + assert(sizeof(DST_KEY) == (TABLE)->key_size); \ + Py_MEMCPY(&(DST_KEY), (PKEY), sizeof(DST_KEY)); \ } while (0) -#define _Py_HASHTABLE_ENTRY_READ_KEY(KEY_SIZE, ENTRY, KEY) \ +#define _Py_HASHTABLE_ENTRY_READ_KEY(TABLE, ENTRY, KEY) \ do { \ - assert(sizeof(KEY) == (KEY_SIZE)); \ - memcpy(&(KEY), _Py_HASHTABLE_ENTRY_KEY(ENTRY), sizeof(KEY)); \ - } while (0) - -#define _Py_HASHTABLE_ENTRY_WRITE_PKEY(KEY_SIZE, ENTRY, PKEY) \ - do { \ - memcpy((void *)_Py_HASHTABLE_ENTRY_KEY(ENTRY), (PKEY), (KEY_SIZE)); \ - } while (0) - -#define _Py_HASHTABLE_ENTRY_READ_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ - do { \ - assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy((PDATA), _Py_HASHTABLE_ENTRY_DATA(TABLE, (ENTRY)), \ - (DATA_SIZE)); \ + assert(sizeof(KEY) == (TABLE)->key_size); \ + Py_MEMCPY(&(KEY), _Py_HASHTABLE_ENTRY_PKEY(ENTRY), sizeof(KEY)); \ } while (0) #define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, ENTRY, DATA) \ - _Py_HASHTABLE_ENTRY_READ_PDATA((TABLE), (ENTRY), sizeof(DATA), &(DATA)) - -#define _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, DATA_SIZE, PDATA) \ do { \ - assert((DATA_SIZE) == (TABLE)->data_size); \ - memcpy((void *)_Py_HASHTABLE_ENTRY_DATA((TABLE), (ENTRY)), \ - (PDATA), (DATA_SIZE)); \ + assert(sizeof(DATA) == (TABLE)->data_size); \ + Py_MEMCPY(&(DATA), _Py_HASHTABLE_ENTRY_PDATA(TABLE, (ENTRY)), \ + sizeof(DATA)); \ } while (0) #define _Py_HASHTABLE_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA) \ - _Py_HASHTABLE_ENTRY_WRITE_PDATA(TABLE, ENTRY, sizeof(DATA), &(DATA)) + do { \ + assert(sizeof(DATA) == (TABLE)->data_size); \ + Py_MEMCPY((void *)_Py_HASHTABLE_ENTRY_PDATA((TABLE), (ENTRY)), \ + &(DATA), sizeof(DATA)); \ + } while (0) /* _Py_hashtable: prototypes */ -typedef Py_uhash_t (*_Py_hashtable_hash_func) (size_t key_size, +/* Forward declaration */ +struct _Py_hashtable_t; + +typedef Py_uhash_t (*_Py_hashtable_hash_func) (struct _Py_hashtable_t *ht, const void *pkey); -typedef int (*_Py_hashtable_compare_func) (size_t key_size, +typedef int (*_Py_hashtable_compare_func) (struct _Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *he); @@ -97,7 +89,7 @@ typedef struct { /* _Py_hashtable: table */ -typedef struct { +typedef struct _Py_hashtable_t { size_t num_buckets; size_t entries; /* Total number of entries in the table. */ _Py_slist_t *buckets; @@ -111,12 +103,12 @@ typedef struct { /* hash a pointer (void*) */ PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr( - size_t key_size, + struct _Py_hashtable_t *ht, const void *pkey); /* comparison using memcmp() */ PyAPI_FUNC(int) _Py_hashtable_compare_direct( - size_t key_size, + _Py_hashtable_t *ht, const void *pkey, const _Py_hashtable_entry_t *entry); diff --git a/Python/marshal.c b/Python/marshal.c index 3be77a8..627a842 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -588,7 +588,7 @@ w_decref_entry(_Py_hashtable_t *ht, _Py_hashtable_entry_t *entry, { PyObject *entry_key; - _Py_HASHTABLE_ENTRY_READ_KEY(ht->key_size, entry, entry_key); + _Py_HASHTABLE_ENTRY_READ_KEY(ht, entry, entry_key); Py_XDECREF(entry_key); return 0; } -- cgit v0.12 From bd31b7c48317eecf981215afbb1f30c81769acbf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 10:32:26 +0100 Subject: Issue #23848: Expose _Py_DumpHexadecimal() This function will be reused by faulthandler. --- Include/traceback.h | 15 ++++++++++++++- Python/traceback.c | 17 +++++++++-------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/Include/traceback.h b/Include/traceback.h index 76e169a..f767ea8 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -94,7 +94,20 @@ PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); /* Format an integer as decimal into the file descriptor fd. This function is signal safe. */ -PyAPI_FUNC(void) _Py_DumpDecimal(int fd, unsigned long value); +PyAPI_FUNC(void) _Py_DumpDecimal( + int fd, + unsigned long value); + +/* Format an integer as hexadecimal into the file descriptor fd with at least + width digits. + + The maximum width is sizeof(unsigned long)*2 digits. + + This function is signal safe. */ +PyAPI_FUNC(void) _Py_DumpHexadecimal( + int fd, + unsigned long value, + Py_ssize_t width); #endif /* !Py_LIMITED_API */ diff --git a/Python/traceback.c b/Python/traceback.c index 403dba5..3259482 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -506,14 +506,15 @@ _Py_DumpDecimal(int fd, unsigned long value) This function is signal safe. */ -static void -dump_hexadecimal(int fd, unsigned long value, Py_ssize_t width) +void +_Py_DumpHexadecimal(int fd, unsigned long value, Py_ssize_t width) { char buffer[sizeof(unsigned long) * 2 + 1], *ptr, *end; const Py_ssize_t size = Py_ARRAY_LENGTH(buffer) - 1; if (width > size) width = size; + /* it's ok if width is negative */ end = &buffer[size]; ptr = end; @@ -582,15 +583,15 @@ _Py_DumpASCII(int fd, PyObject *text) } else if (ch <= 0xff) { PUTS(fd, "\\x"); - dump_hexadecimal(fd, ch, 2); + _Py_DumpHexadecimal(fd, ch, 2); } else if (ch <= 0xffff) { PUTS(fd, "\\u"); - dump_hexadecimal(fd, ch, 4); + _Py_DumpHexadecimal(fd, ch, 4); } else { PUTS(fd, "\\U"); - dump_hexadecimal(fd, ch, 8); + _Py_DumpHexadecimal(fd, ch, 8); } } if (truncated) { @@ -693,9 +694,9 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) PUTS(fd, "Current thread 0x"); else PUTS(fd, "Thread 0x"); - dump_hexadecimal(fd, - (unsigned long)tstate->thread_id, - sizeof(unsigned long) * 2); + _Py_DumpHexadecimal(fd, + (unsigned long)tstate->thread_id, + sizeof(unsigned long) * 2); PUTS(fd, " (most recent call first):\n"); } -- cgit v0.12 From 404cdc5a924a294a17c04b745df5549c32e08130 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 10:39:17 +0100 Subject: faulthandler: add Windows exception handler Issue #23848: On Windows, faulthandler.enable() now also installs an exception handler to dump the traceback of all Python threads on any Windows exception, not only on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). --- Doc/library/faulthandler.rst | 3 + Doc/whatsnew/3.6.rst | 8 ++ Lib/test/test_faulthandler.py | 62 ++++++++--- Misc/NEWS | 4 + Modules/faulthandler.c | 253 +++++++++++++++++++++++++++++++----------- 5 files changed, 253 insertions(+), 77 deletions(-) diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 3a5badd..3c49649 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -68,6 +68,9 @@ Fault handler state .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.6 + On Windows, a handler for Windows exception is also installed. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 986c145..928d748 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -199,6 +199,14 @@ directives ``%G``, ``%u`` and ``%V``. (Contributed by Ashley Anderson in :issue:`12006`.) +faulthandler +------------ + +On Windows, the :mod:`faulthandler` module now installs an handler for Windows +exceptions: see :func:`faulthandler.enable`. (Contributed by Victor Stinner in +:issue:`23848`.) + + os -- diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index c3cd657..fbd535b 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -23,6 +23,7 @@ except ImportError: _testcapi = None TIMEOUT = 0.5 +MS_WINDOWS = (os.name == 'nt') def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header @@ -76,9 +77,9 @@ class FaultHandlerTests(unittest.TestCase): output = output.decode('ascii', 'backslashreplace') return output.splitlines(), exitcode - def check_fatal_error(self, code, line_number, name_regex, - filename=None, all_threads=True, other_regex=None, - fd=None, know_current_thread=True): + def check_error(self, code, line_number, fatal_error, *, + filename=None, all_threads=True, other_regex=None, + fd=None, know_current_thread=True): """ Check that the fault handler for fatal errors is enabled and check the traceback from the child process output. @@ -93,14 +94,14 @@ class FaultHandlerTests(unittest.TestCase): else: header = 'Stack' regex = """ - ^Fatal Python error: {name} + ^{fatal_error} {header} \(most recent call first\): File "", line {lineno} in """ regex = dedent(regex.format( lineno=line_number, - name=name_regex, + fatal_error=fatal_error, header=header)).strip() if other_regex: regex += '|' + other_regex @@ -109,17 +110,36 @@ class FaultHandlerTests(unittest.TestCase): self.assertRegex(output, regex) self.assertNotEqual(exitcode, 0) + def check_fatal_error(self, code, line_number, name_regex, **kw): + fatal_error = 'Fatal Python error: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + + def check_windows_exception(self, code, line_number, name_regex, **kw): + fatal_error = 'Windows exception: %s' % name_regex + self.check_error(code, line_number, fatal_error, **kw) + @unittest.skipIf(sys.platform.startswith('aix'), "the first page of memory is a mapped read-only on AIX") def test_read_null(self): - self.check_fatal_error(""" - import faulthandler - faulthandler.enable() - faulthandler._read_null() - """, - 3, - # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion - '(?:Segmentation fault|Bus error|Illegal instruction)') + if not MS_WINDOWS: + self.check_fatal_error(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion + '(?:Segmentation fault' + '|Bus error' + '|Illegal instruction)') + else: + self.check_windows_exception(""" + import faulthandler + faulthandler.enable() + faulthandler._read_null() + """, + 3, + 'access violation') def test_sigsegv(self): self.check_fatal_error(""" @@ -708,6 +728,22 @@ class FaultHandlerTests(unittest.TestCase): with self.check_stderr_none(): faulthandler.register(signal.SIGUSR1) + @unittest.skipUnless(MS_WINDOWS, 'specific to Windows') + def test_raise_exception(self): + for exc, name in ( + ('EXCEPTION_ACCESS_VIOLATION', 'access violation'), + ('EXCEPTION_INT_DIVIDE_BY_ZERO', 'int divide by zero'), + ('EXCEPTION_STACK_OVERFLOW', 'stack overflow'), + ): + self.check_windows_exception(f""" + import faulthandler + faulthandler.enable() + faulthandler._raise_exception(faulthandler._{exc}) + """, + 3, + name) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS b/Misc/NEWS index 29fc65d..70ff3de 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,10 @@ Core and Builtins Library ------- +- Issue #23848: On Windows, faulthandler.enable() now also installs an + exception handler to dump the traceback of all Python threads on any Windows + exception, not only on UNIX signals (SIGSEGV, SIGFPE, SIGABRT). + - Issue #26530: Add C functions :c:func:`_PyTraceMalloc_Track` and :c:func:`_PyTraceMalloc_Untrack` to track memory blocks using the :mod:`tracemalloc` module. Add :c:func:`_PyTraceMalloc_GetTraceback` to get diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1c247b7..2214466 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -119,7 +119,7 @@ static fault_handler_t faulthandler_handlers[] = { handler fails in faulthandler_fatal_error() */ {SIGSEGV, 0, "Segmentation fault", } }; -static const unsigned char faulthandler_nsignals = \ +static const size_t faulthandler_nsignals = \ Py_ARRAY_LENGTH(faulthandler_handlers); #ifdef HAVE_SIGALTSTACK @@ -290,6 +290,19 @@ faulthandler_dump_traceback_py(PyObject *self, Py_RETURN_NONE; } +static void +faulthandler_disable_fatal_handler(fault_handler_t *handler) +{ + if (!handler->enabled) + return; + handler->enabled = 0; +#ifdef HAVE_SIGACTION + (void)sigaction(handler->signum, &handler->previous, NULL); +#else + (void)signal(handler->signum, handler->previous); +#endif +} + /* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. @@ -308,7 +321,7 @@ static void faulthandler_fatal_error(int signum) { const int fd = fatal_error.fd; - unsigned int i; + size_t i; fault_handler_t *handler = NULL; int save_errno = errno; @@ -326,12 +339,7 @@ faulthandler_fatal_error(int signum) } /* restore the previous handler */ -#ifdef HAVE_SIGACTION - (void)sigaction(signum, &handler->previous, NULL); -#else - (void)signal(signum, handler->previous); -#endif - handler->enabled = 0; + faulthandler_disable_fatal_handler(handler); PUTS(fd, "Fatal Python error: "); PUTS(fd, handler->name); @@ -353,20 +361,110 @@ faulthandler_fatal_error(int signum) raise(signum); } +#ifdef MS_WINDOWS +static LONG WINAPI +faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) +{ + const int fd = fatal_error.fd; + DWORD code = exc_info->ExceptionRecord->ExceptionCode; + + PUTS(fd, "Windows exception: "); + switch (code) + { + /* only format most common errors */ + case EXCEPTION_ACCESS_VIOLATION: PUTS(fd, "access violation"); break; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: PUTS(fd, "float divide by zero"); break; + case EXCEPTION_FLT_OVERFLOW: PUTS(fd, "float overflow"); break; + case EXCEPTION_INT_DIVIDE_BY_ZERO: PUTS(fd, "int divide by zero"); break; + case EXCEPTION_INT_OVERFLOW: PUTS(fd, "integer overflow"); break; + case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; + case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; + default: + PUTS(fd, "code 0x"); + _Py_DumpHexadecimal(fd, code, sizeof(DWORD)); + } + PUTS(fd, "\n\n"); + + if (code == EXCEPTION_ACCESS_VIOLATION) { + /* disable signal handler for SIGSEGV */ + size_t i; + for (i=0; i < faulthandler_nsignals; i++) { + fault_handler_t *handler = &faulthandler_handlers[i]; + if (handler->signum == SIGSEGV) { + faulthandler_disable_fatal_handler(handler); + break; + } + } + } + + faulthandler_dump_traceback(fd, fatal_error.all_threads, + fatal_error.interp); + + /* call the next exception handler */ + return EXCEPTION_CONTINUE_SEARCH; +} +#endif + /* Install the handler for fatal signals, faulthandler_fatal_error(). */ -static PyObject* -faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) +int +faulthandler_enable(void) { - static char *kwlist[] = {"file", "all_threads", NULL}; - PyObject *file = NULL; - int all_threads = 1; - unsigned int i; + size_t i; fault_handler_t *handler; #ifdef HAVE_SIGACTION struct sigaction action; #endif int err; + + if (fatal_error.enabled) { + return 0; + } + + fatal_error.enabled = 1; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + +#ifdef HAVE_SIGACTION + action.sa_handler = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER; +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + err = sigaction(handler->signum, &action, &handler->previous); +#else + handler->previous = signal(handler->signum, + faulthandler_fatal_error); + err = (handler->previous == SIG_ERR); +#endif + if (err) { + PyErr_SetFromErrno(PyExc_RuntimeError); + return -1; + } + + handler->enabled = 1; + } + +#ifdef MS_WINDOWS + AddVectoredExceptionHandler(1, faulthandler_exc_handler); +#endif + return 0; +} + +static PyObject* +faulthandler_py_enable(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; int fd; PyThreadState *tstate; @@ -388,37 +486,10 @@ faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) fatal_error.all_threads = all_threads; fatal_error.interp = tstate->interp; - if (!fatal_error.enabled) { - fatal_error.enabled = 1; - - for (i=0; i < faulthandler_nsignals; i++) { - handler = &faulthandler_handlers[i]; -#ifdef HAVE_SIGACTION - action.sa_handler = faulthandler_fatal_error; - sigemptyset(&action.sa_mask); - /* Do not prevent the signal from being received from within - its own signal handler */ - action.sa_flags = SA_NODEFER; -#ifdef HAVE_SIGALTSTACK - if (stack.ss_sp != NULL) { - /* Call the signal handler on an alternate signal stack - provided by sigaltstack() */ - action.sa_flags |= SA_ONSTACK; - } -#endif - err = sigaction(handler->signum, &action, &handler->previous); -#else - handler->previous = signal(handler->signum, - faulthandler_fatal_error); - err = (handler->previous == SIG_ERR); -#endif - if (err) { - PyErr_SetFromErrno(PyExc_RuntimeError); - return NULL; - } - handler->enabled = 1; - } + if (faulthandler_enable() < 0) { + return NULL; } + Py_RETURN_NONE; } @@ -432,14 +503,7 @@ faulthandler_disable(void) fatal_error.enabled = 0; for (i=0; i < faulthandler_nsignals; i++) { handler = &faulthandler_handlers[i]; - if (!handler->enabled) - continue; -#ifdef HAVE_SIGACTION - (void)sigaction(handler->signum, &handler->previous, NULL); -#else - (void)signal(handler->signum, handler->previous); -#endif - handler->enabled = 0; + faulthandler_disable_fatal_handler(handler); } } @@ -991,7 +1055,10 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) Py_RETURN_NONE; } + #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +#define FAULTHANDLER_STACK_OVERFLOW + #ifdef __INTEL_COMPILER /* Issue #23654: Turn off ICC's tail call optimization for the * stack_overflow generator. ICC turns the recursive tail call into @@ -1005,12 +1072,21 @@ stack_overflow(Py_uintptr_t min_sp, Py_uintptr_t max_sp, size_t *depth) /* allocate 4096 bytes on the stack at each call */ unsigned char buffer[4096]; Py_uintptr_t sp = (Py_uintptr_t)&buffer; + Py_uintptr_t stop; + *depth += 1; - if (sp < min_sp || max_sp < sp) + if (sp < min_sp || max_sp < sp) { + printf("call #%lu\n", (unsigned long)*depth); return sp; - buffer[0] = 1; - buffer[4095] = 0; - return stack_overflow(min_sp, max_sp, depth); + } + + memset(buffer, (unsigned char)*depth, sizeof(buffer)); + stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; + + memset(buffer, (unsigned char)stop, sizeof(buffer)); + stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; + + return stop; } static PyObject * @@ -1018,13 +1094,19 @@ faulthandler_stack_overflow(PyObject *self) { size_t depth, size; Py_uintptr_t sp = (Py_uintptr_t)&depth; - Py_uintptr_t stop; + Py_uintptr_t min_sp, max_sp, stop; faulthandler_suppress_crash_report(); + depth = 0; - stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, - sp + STACK_OVERFLOW_MAX_SIZE, - &depth); + if (sp > STACK_OVERFLOW_MAX_SIZE) + min_sp = sp - STACK_OVERFLOW_MAX_SIZE; + else + min_sp = 0; + max_sp = sp + STACK_OVERFLOW_MAX_SIZE; + + stop = stack_overflow(min_sp, max_sp, &depth); + if (sp < stop) size = stop - sp; else @@ -1035,7 +1117,7 @@ faulthandler_stack_overflow(PyObject *self) size, depth); return NULL; } -#endif +#endif /* (defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION)) ... */ static int @@ -1058,12 +1140,25 @@ faulthandler_traverse(PyObject *module, visitproc visit, void *arg) return 0; } +#ifdef MS_WINDOWS +static PyObject * +faulthandler_raise_exception(PyObject *self, PyObject *args) +{ + unsigned int code, flags = 0; + if (!PyArg_ParseTuple(args, "I|I:_raise_exception", &code, &flags)) + return NULL; + faulthandler_suppress_crash_report(); + RaiseException(code, flags, 0, NULL); + Py_RETURN_NONE; +} +#endif + PyDoc_STRVAR(module_doc, "faulthandler module."); static PyMethodDef module_methods[] = { {"enable", - (PyCFunction)faulthandler_enable, METH_VARARGS|METH_KEYWORDS, + (PyCFunction)faulthandler_py_enable, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("enable(file=sys.stderr, all_threads=True): " "enable the fault handler")}, {"disable", (PyCFunction)faulthandler_disable_py, METH_NOARGS, @@ -1117,10 +1212,14 @@ static PyMethodDef module_methods[] = { PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, {"_fatal_error", faulthandler_fatal_error_py, METH_VARARGS, PyDoc_STR("_fatal_error(message): call Py_FatalError(message)")}, -#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +#ifdef FAULTHANDLER_STACK_OVERFLOW {"_stack_overflow", (PyCFunction)faulthandler_stack_overflow, METH_NOARGS, PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, #endif +#ifdef MS_WINDOWS + {"_raise_exception", faulthandler_raise_exception, METH_VARARGS, + PyDoc_STR("raise_exception(code, flags=0): Call RaiseException(code, flags).")}, +#endif {NULL, NULL} /* sentinel */ }; @@ -1139,7 +1238,33 @@ static struct PyModuleDef module_def = { PyMODINIT_FUNC PyInit_faulthandler(void) { - return PyModule_Create(&module_def); + PyObject *m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + + /* Add constants for unit tests */ +#ifdef MS_WINDOWS + /* RaiseException() codes (prefixed by an underscore) */ + if (PyModule_AddIntConstant(m, "_EXCEPTION_ACCESS_VIOLATION", + EXCEPTION_ACCESS_VIOLATION)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_INT_DIVIDE_BY_ZERO", + EXCEPTION_INT_DIVIDE_BY_ZERO)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_STACK_OVERFLOW", + EXCEPTION_STACK_OVERFLOW)) + return NULL; + + /* RaiseException() flags (prefixed by an underscore) */ + if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE", + EXCEPTION_NONCONTINUABLE)) + return NULL; + if (PyModule_AddIntConstant(m, "_EXCEPTION_NONCONTINUABLE_EXCEPTION", + EXCEPTION_NONCONTINUABLE_EXCEPTION)) + return NULL; +#endif + + return m; } /* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable -- cgit v0.12 From 0aed3a4ebc42afcf41ea926b5fca721a0923b43e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:30:43 +0100 Subject: _PyMem_DebugFree(): fix compiler warning on Windows Don't return a void value. --- Objects/obmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 503fcdf..40c9fcd 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2039,7 +2039,7 @@ static void _PyMem_DebugFree(void *ctx, void *ptr) { _PyMem_DebugCheckGIL(); - return _PyMem_DebugRawFree(ctx, ptr); + _PyMem_DebugRawFree(ctx, ptr); } static void * -- cgit v0.12 From ccb1f8cb1a9faabe3a3ef90b64257f632052c368 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:31:58 +0100 Subject: getpathp.c: fix compiler warning wcsnlen_s() result type is size_t. --- PC/getpathp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PC/getpathp.c b/PC/getpathp.c index c7ddf1e..cb9f1db 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -135,7 +135,7 @@ exists(wchar_t *filename) static int ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc/.pyo too */ { - int n; + size_t n; if (exists(filename)) return 1; @@ -553,7 +553,7 @@ calculate_path(void) envpath = NULL; pythonhome = argv0_path; } - + /* Look for a 'home' variable and set argv0_path to it, if found */ if (find_env_config_value(env_file, L"home", tmpbuffer)) { wcscpy_s(argv0_path, MAXPATHLEN+1, tmpbuffer); -- cgit v0.12 From 976bb4099c41af2eda00213eeac3083829f12c98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:36:19 +0100 Subject: compiler.c: fix compiler warnings on Windows --- Python/compile.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index e86b293..92fea3d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -195,7 +195,7 @@ static int expr_constant(struct compiler *, expr_ty); static int compiler_with(struct compiler *, stmt_ty, int); static int compiler_async_with(struct compiler *, stmt_ty, int); static int compiler_async_for(struct compiler *, stmt_ty); -static int compiler_call_helper(struct compiler *c, Py_ssize_t n, +static int compiler_call_helper(struct compiler *c, int n, asdl_seq *args, asdl_seq *keywords); static int compiler_try_except(struct compiler *, stmt_ty); @@ -476,9 +476,9 @@ compiler_unit_check(struct compiler_unit *u) { basicblock *block; for (block = u->u_blocks; block != NULL; block = block->b_list) { - assert((void *)block != (void *)0xcbcbcbcb); - assert((void *)block != (void *)0xfbfbfbfb); - assert((void *)block != (void *)0xdbdbdbdb); + assert((Py_uintptr_t)block != 0xcbcbcbcbU); + assert((Py_uintptr_t)block != 0xfbfbfbfbU); + assert((Py_uintptr_t)block != 0xdbdbdbdbU); if (block->b_instr != NULL) { assert(block->b_ialloc > 0); assert(block->b_iused > 0); @@ -3097,7 +3097,8 @@ compiler_set(struct compiler *c, expr_ty e) static int compiler_dict(struct compiler *c, expr_ty e) { - Py_ssize_t i, n, containers, elements; + Py_ssize_t i, n, elements; + int containers; int is_unpacking = 0; n = asdl_seq_LEN(e->v.Dict.values); containers = 0; @@ -3267,12 +3268,13 @@ compiler_formatted_value(struct compiler *c, expr_ty e) /* shared code between compiler_call and compiler_class */ static int compiler_call_helper(struct compiler *c, - Py_ssize_t n, /* Args already pushed */ + int n, /* Args already pushed */ asdl_seq *args, asdl_seq *keywords) { int code = 0; - Py_ssize_t nelts, i, nseen, nkw; + Py_ssize_t nelts, i, nseen; + int nkw; /* the number of tuples and dictionaries on the stack */ Py_ssize_t nsubargs = 0, nsubkwargs = 0; -- cgit v0.12 From e985726553b4e5571f110f00e14da5a6ae5d7994 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 11:37:41 +0100 Subject: _msi.c: try to fix compiler warnings --- PC/_msi.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/PC/_msi.c b/PC/_msi.c index 86a5943..b66b18b 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -1032,12 +1032,12 @@ PyInit__msi(void) if (m == NULL) return NULL; - PyModule_AddIntConstant(m, "MSIDBOPEN_CREATEDIRECT", (int)MSIDBOPEN_CREATEDIRECT); - PyModule_AddIntConstant(m, "MSIDBOPEN_CREATE", (int)MSIDBOPEN_CREATE); - PyModule_AddIntConstant(m, "MSIDBOPEN_DIRECT", (int)MSIDBOPEN_DIRECT); - PyModule_AddIntConstant(m, "MSIDBOPEN_READONLY", (int)MSIDBOPEN_READONLY); - PyModule_AddIntConstant(m, "MSIDBOPEN_TRANSACT", (int)MSIDBOPEN_TRANSACT); - PyModule_AddIntConstant(m, "MSIDBOPEN_PATCHFILE", (int)MSIDBOPEN_PATCHFILE); + PyModule_AddIntConstant(m, "MSIDBOPEN_CREATEDIRECT", (long)MSIDBOPEN_CREATEDIRECT); + PyModule_AddIntConstant(m, "MSIDBOPEN_CREATE", (long)MSIDBOPEN_CREATE); + PyModule_AddIntConstant(m, "MSIDBOPEN_DIRECT", (long)MSIDBOPEN_DIRECT); + PyModule_AddIntConstant(m, "MSIDBOPEN_READONLY", (long)MSIDBOPEN_READONLY); + PyModule_AddIntConstant(m, "MSIDBOPEN_TRANSACT", (long)MSIDBOPEN_TRANSACT); + PyModule_AddIntConstant(m, "MSIDBOPEN_PATCHFILE", (long)MSIDBOPEN_PATCHFILE); PyModule_AddIntMacro(m, MSICOLINFO_NAMES); PyModule_AddIntMacro(m, MSICOLINFO_TYPES); -- cgit v0.12 From 69649f21f02907da59264df8faf73849d2557f5e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 12:14:10 +0100 Subject: regrtest: display test duration in sequential mode Only display duration if a test takes more than 30 seconds. --- Lib/test/libregrtest/main.py | 21 +++++++++++++++++++-- Lib/test/libregrtest/runtest.py | 5 +++++ Lib/test/libregrtest/runtest_mp.py | 7 ++----- Lib/test/test_regrtest.py | 5 +++-- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 1c99f2b..b954db5 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -13,7 +13,8 @@ from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR) + INTERRUPTED, CHILD_ERROR, + PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests from test import support try: @@ -293,8 +294,15 @@ class Regrtest: save_modules = sys.modules.keys() + previous_test = None for test_index, test in enumerate(self.tests, 1): - self.display_progress(test_index, test) + start_time = time.monotonic() + + text = test + if previous_test: + text = '%s -- %s' % (text, previous_test) + self.display_progress(test_index, text) + if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. @@ -311,6 +319,12 @@ class Regrtest: else: self.accumulate_result(test, result) + test_time = time.monotonic() - start_time + if test_time >= PROGRESS_MIN_TIME: + previous_test = '%s took %.0f sec' % (test, test_time) + else: + previous_test = None + if self.ns.findleaks: gc.collect() if gc.garbage: @@ -326,6 +340,9 @@ class Regrtest: if module not in save_modules and module.startswith("test."): support.unload(module) + if previous_test: + print(previous_test) + def _test_forever(self, tests): while True: for test in tests: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 043f23c..daff476 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -20,6 +20,11 @@ RESOURCE_DENIED = -3 INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + + # small set of tests to determine if we have a basically functioning interpreter # (i.e. if any of these fail, then anything else is likely to follow) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 5e847a0..e51b100 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -13,14 +13,11 @@ except ImportError: print("Multiprocess option requires thread support") sys.exit(2) -from test.libregrtest.runtest import runtest, INTERRUPTED, CHILD_ERROR +from test.libregrtest.runtest import ( + runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME) from test.libregrtest.setup import setup_tests -# Minimum duration of a test to display its duration or to mention that -# the test is running in background -PROGRESS_MIN_TIME = 30.0 # seconds - # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 18d0f46..03e7e1d 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -304,7 +304,7 @@ class ParseArgsTestCase(unittest.TestCase): class BaseTestCase(unittest.TestCase): TEST_UNIQUE_ID = 1 TESTNAME_PREFIX = 'test_regrtest_' - TESTNAME_REGEX = r'test_[a-z0-9_]+' + TESTNAME_REGEX = r'test_[a-zA-Z0-9_]+' def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) @@ -351,7 +351,8 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)$' % self.TESTNAME_REGEX + regex = (r'^[0-9]+:[0-9]+:[0-9]+ \[ *[0-9]+(?:/ *[0-9]+)?\] (%s)' + % self.TESTNAME_REGEX) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) -- cgit v0.12 From 96f6e7a1edfdbfbb85538d897ec6001f218b351e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 12:38:01 +0100 Subject: Buildbots: change also Windows timeout from 1 hour to 15 min --- Tools/buildbot/test.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index ff7d167..c01400c 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -16,4 +16,4 @@ if "%1"=="+q" (set rt_opts=%rt_opts:-q=%) & shift & goto CheckOpts if NOT "%1"=="" (set regrtest_args=%regrtest_args% %1) & shift & goto CheckOpts echo on -call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=3600 %regrtest_args% +call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --timeout=900 %regrtest_args% -- cgit v0.12 From 412a5e7e2349689ab408c9475401c83886108e75 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 14:44:14 +0100 Subject: faulthandler: only log fatal exceptions Issue #23848, #26622: * faulthandler now only logs fatal Windows exceptions. * write error code as decimal, not as hexadecimal * replace "Windows exception" with "Windows fatal exception" --- Lib/test/test_faulthandler.py | 2 +- Modules/faulthandler.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index fbd535b..a3a2116 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -115,7 +115,7 @@ class FaultHandlerTests(unittest.TestCase): self.check_error(code, line_number, fatal_error, **kw) def check_windows_exception(self, code, line_number, name_regex, **kw): - fatal_error = 'Windows exception: %s' % name_regex + fatal_error = 'Windows fatal exception: %s' % name_regex self.check_error(code, line_number, fatal_error, **kw) @unittest.skipIf(sys.platform.startswith('aix'), diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 2214466..e86f2bb 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -367,8 +367,15 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) { const int fd = fatal_error.fd; DWORD code = exc_info->ExceptionRecord->ExceptionCode; + DWORD flags = exc_info->ExceptionRecord->ExceptionFlags; - PUTS(fd, "Windows exception: "); + /* only log fatal exceptions */ + if (flags & EXCEPTION_NONCONTINUABLE) { + /* call the next exception handler */ + return EXCEPTION_CONTINUE_SEARCH; + } + + PUTS(fd, "Windows fatal exception: "); switch (code) { /* only format most common errors */ @@ -380,8 +387,8 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_IN_PAGE_ERROR: PUTS(fd, "page error"); break; case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: - PUTS(fd, "code 0x"); - _Py_DumpHexadecimal(fd, code, sizeof(DWORD)); + PUTS(fd, "code "); + _Py_DumpDecimal(fd, code, sizeof(DWORD)); } PUTS(fd, "\n\n"); -- cgit v0.12 From 928ad285b5eb0f25d1dabd48ff3761c205efb6f5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 15:19:12 +0100 Subject: Issue #23848: Try to fix test_faulthandler on ARM Restore the previous code for stack_overflow(). --- Modules/faulthandler.c | 48 ++++++++++++++++-------------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index e86f2bb..03afe0e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -418,21 +418,21 @@ int faulthandler_enable(void) { size_t i; - fault_handler_t *handler; -#ifdef HAVE_SIGACTION - struct sigaction action; -#endif - int err; if (fatal_error.enabled) { return 0; } - fatal_error.enabled = 1; for (i=0; i < faulthandler_nsignals; i++) { - handler = &faulthandler_handlers[i]; + fault_handler_t *handler; +#ifdef HAVE_SIGACTION + struct sigaction action; +#endif + int err; + handler = &faulthandler_handlers[i]; + assert(!handler->enabled); #ifdef HAVE_SIGACTION action.sa_handler = faulthandler_fatal_error; sigemptyset(&action.sa_mask); @@ -1062,7 +1062,6 @@ faulthandler_fatal_error_py(PyObject *self, PyObject *args) Py_RETURN_NONE; } - #if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) #define FAULTHANDLER_STACK_OVERFLOW @@ -1079,21 +1078,12 @@ stack_overflow(Py_uintptr_t min_sp, Py_uintptr_t max_sp, size_t *depth) /* allocate 4096 bytes on the stack at each call */ unsigned char buffer[4096]; Py_uintptr_t sp = (Py_uintptr_t)&buffer; - Py_uintptr_t stop; - *depth += 1; - if (sp < min_sp || max_sp < sp) { - printf("call #%lu\n", (unsigned long)*depth); + if (sp < min_sp || max_sp < sp) return sp; - } - - memset(buffer, (unsigned char)*depth, sizeof(buffer)); - stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; - - memset(buffer, (unsigned char)stop, sizeof(buffer)); - stop = stack_overflow(min_sp, max_sp, depth) + buffer[0]; - - return stop; + buffer[0] = 1; + buffer[4095] = 0; + return stack_overflow(min_sp, max_sp, depth); } static PyObject * @@ -1101,19 +1091,13 @@ faulthandler_stack_overflow(PyObject *self) { size_t depth, size; Py_uintptr_t sp = (Py_uintptr_t)&depth; - Py_uintptr_t min_sp, max_sp, stop; + Py_uintptr_t stop; faulthandler_suppress_crash_report(); - depth = 0; - if (sp > STACK_OVERFLOW_MAX_SIZE) - min_sp = sp - STACK_OVERFLOW_MAX_SIZE; - else - min_sp = 0; - max_sp = sp + STACK_OVERFLOW_MAX_SIZE; - - stop = stack_overflow(min_sp, max_sp, &depth); - + stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, + sp + STACK_OVERFLOW_MAX_SIZE, + &depth); if (sp < stop) size = stop - sp; else @@ -1124,7 +1108,7 @@ faulthandler_stack_overflow(PyObject *self) size, depth); return NULL; } -#endif /* (defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION)) ... */ +#endif /* defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) */ static int -- cgit v0.12 From 1c3069aed6098804f3ce4c810e07dbd02fc797c8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 16:10:07 +0100 Subject: Rework _Py_DumpASCII() to make Coverity happy --- Python/traceback.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/traceback.c b/Python/traceback.c index 3259482..62a6b1e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -545,23 +545,23 @@ _Py_DumpASCII(int fd, PyObject *text) size = ascii->length; kind = ascii->state.kind; - if (ascii->state.compact) { + if (kind == PyUnicode_WCHAR_KIND) { + wstr = ((PyASCIIObject *)text)->wstr; + if (wstr == NULL) + return; + size = ((PyCompactUnicodeObject *)text)->wstr_length; + } + else if (ascii->state.compact) { if (ascii->state.ascii) data = ((PyASCIIObject*)text) + 1; else data = ((PyCompactUnicodeObject*)text) + 1; } - else if (kind != PyUnicode_WCHAR_KIND) { + else { data = ((PyUnicodeObject *)text)->data.any; if (data == NULL) return; } - else { - wstr = ((PyASCIIObject *)text)->wstr; - if (wstr == NULL) - return; - size = ((PyCompactUnicodeObject *)text)->wstr_length; - } if (MAX_STRING_LENGTH < size) { size = MAX_STRING_LENGTH; -- cgit v0.12 From d1700a9360e55f05b2dec5654353bbd947ba19e8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 16:57:51 +0100 Subject: Fix typo in doc: avoid the french "& cie" :-) --- Doc/c-api/memory.rst | 2 +- Doc/using/cmdline.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index c2e2442..bd0bc12 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -378,7 +378,7 @@ with a fixed size of 256 KB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_OBJ` domain -(:c:func:`PyObject_Malloc` & cie). +(ex: :c:func:`PyObject_Malloc`). The arena allocator uses the following functions: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7ff9361..c1c700c 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -628,12 +628,12 @@ conflict. Set the family of memory allocators used by Python: * ``malloc``: use the :c:func:`malloc` function of the C library - for all Python memory allocators (:c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_Malloc`, :c:func:`PyObject_Malloc` & cie). + for all Python memory allocators (ex: :c:func:`PyMem_RawMalloc`, + :c:func:`PyMem_Malloc` and :c:func:`PyObject_Malloc`). * ``pymalloc``: :c:func:`PyObject_Malloc`, :c:func:`PyObject_Calloc` and :c:func:`PyObject_Realloc` use the :ref:`pymalloc allocator `. - Other Python memory allocators (:c:func:`PyMem_RawMalloc`, - :c:func:`PyMem_Malloc` & cie) use :c:func:`malloc`. + Other Python memory allocators (ex: :c:func:`PyMem_RawMalloc` and + :c:func:`PyMem_Malloc`) use :c:func:`malloc`. Install debug hooks: -- cgit v0.12 From 023654fa68c73e34da01610cad330f98d2db79cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 17:48:22 +0100 Subject: get_warnings_attr(): Fix coverity warning Don't check if the dict key exists before getting the key. Instead get the key and handle error. --- Python/_warnings.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 41eaf53..40f5c8e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -45,7 +45,6 @@ get_warnings_attr(const char *attr, int try_import) static PyObject *warnings_str = NULL; PyObject *all_modules; PyObject *warnings_module, *obj; - int result; if (warnings_str == NULL) { warnings_str = PyUnicode_InternFromString("warnings"); @@ -65,11 +64,11 @@ get_warnings_attr(const char *attr, int try_import) } else { all_modules = PyImport_GetModuleDict(); - result = PyDict_Contains(all_modules, warnings_str); - if (result == -1 || result == 0) - return NULL; warnings_module = PyDict_GetItem(all_modules, warnings_str); + if (warnings_module == NULL) + return NULL; + Py_INCREF(warnings_module); } -- cgit v0.12 From 904f5def5cc6da106a1e2e9feb0830cdb1433519 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:32:54 +0100 Subject: Try to fix test_gdb on s390x buildbots --- Lib/test/test_gdb.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 3fe15e4..8e3ccb0 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -199,25 +199,13 @@ class DebuggerTests(unittest.TestCase): # Ignore some benign messages on stderr. ignore_patterns = ( 'Function "%s" not defined.' % breakpoint, - "warning: no loadable sections found in added symbol-file" - " system-supplied DSO", - "warning: Unable to find libthread_db matching" - " inferior's thread library, thread debugging will" - " not be available.", - "warning: Cannot initialize thread debugging" - " library: Debugger service failed", - 'warning: Could not load shared library symbols for ' - 'linux-vdso.so', - 'warning: Could not load shared library symbols for ' - 'linux-gate.so', - 'warning: Could not load shared library symbols for ' - 'linux-vdso64.so', 'Do you need "set solib-search-path" or ' '"set sysroot"?', - 'warning: Source file is more recent than executable.', - # Issue #19753: missing symbols on System Z - 'Missing separate debuginfo for ', - 'Try: zypper install -C ', + # BFD: /usr/lib/debug/(...): unable to initialize decompress + # status for section .debug_aranges + 'BFD: ', + # ignore all warnings + 'warning: ', ) for line in errlines: if not line.startswith(ignore_patterns): -- cgit v0.12 From f963c135979838893772619e83f94db97c536048 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:36:54 +0100 Subject: longobject.c: fix compilation warning on Windows 64-bit We know that Py_SIZE(b) is -1 or 1 an so fits into the sdigit type. --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 3f9837f..70d8cfc 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3522,7 +3522,7 @@ fast_mod(PyLongObject *a, PyLongObject *b) mod = right - 1 - (left - 1) % right; } - return PyLong_FromLong(mod * Py_SIZE(b)); + return PyLong_FromLong(mod * (sdigit)Py_SIZE(b)); } /* Fast floor division for single-digit longs. */ -- cgit v0.12 From 82d44f0598a38034174dae4416c315d2c3926456 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:37:54 +0100 Subject: Issue #23848: Fix usage of _Py_DumpDecimal() --- Modules/faulthandler.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 03afe0e..a990d25 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -388,7 +388,7 @@ faulthandler_exc_handler(struct _EXCEPTION_POINTERS *exc_info) case EXCEPTION_STACK_OVERFLOW: PUTS(fd, "stack overflow"); break; default: PUTS(fd, "code "); - _Py_DumpDecimal(fd, code, sizeof(DWORD)); + _Py_DumpDecimal(fd, code); } PUTS(fd, "\n\n"); -- cgit v0.12 From 4a1c7d2c3699a0f201ec2bb277ebcadf3f863759 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 18:45:55 +0100 Subject: Try to fix test_spwd on OpenIndiana Issue #18787: try to get the "root" entry which should exist on all UNIX instead of "bin" which doesn't exist on OpenIndiana. --- Lib/test/test_spwd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index fca809e..3a11a2d 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -62,7 +62,7 @@ class TestSpwdNonRoot(unittest.TestCase): def test_getspnam_exception(self): with self.assertRaises(PermissionError) as cm: - spwd.getspnam('bin') + spwd.getspnam('root') self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') -- cgit v0.12 From 66e9d03bf445b595861a3749df75b269ba3634e4 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Wed, 23 Mar 2016 20:50:10 +0100 Subject: Issue #26621: Update libmpdec version and remove unnecessary test case. --- Lib/_pydecimal.py | 2 +- Lib/test/test_decimal.py | 1 - Modules/_decimal/libmpdec/mpdecimal.h | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 02365ca..60723f3 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -148,7 +148,7 @@ __xname__ = __name__ # sys.modules lookup (--without-threads) __name__ = 'decimal' # For pickling __version__ = '1.70' # Highest version of the spec this complies with # See http://speleotrove.com/decimal/ -__libmpdec_version__ = "2.4.1" # compatible libmpdec version +__libmpdec_version__ = "2.4.2" # compatible libmpdec version import math as _math import numbers as _numbers diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index f6d58ff..f56f9ad 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4206,7 +4206,6 @@ class CheckAttributes(unittest.TestCase): self.assertTrue(P.HAVE_THREADS is True or P.HAVE_THREADS is False) self.assertEqual(C.__version__, P.__version__) - self.assertEqual(C.__libmpdec_version__, P.__libmpdec_version__) self.assertEqual(dir(C), dir(P)) diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h index 5ca7413..56e4887 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.h +++ b/Modules/_decimal/libmpdec/mpdecimal.h @@ -108,9 +108,9 @@ MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) #define MPD_MAJOR_VERSION 2 #define MPD_MINOR_VERSION 4 -#define MPD_MICRO_VERSION 1 +#define MPD_MICRO_VERSION 2 -#define MPD_VERSION "2.4.1" +#define MPD_VERSION "2.4.2" #define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \ (MPD_MINOR_VERSION << 16) | \ -- cgit v0.12 From c53195bbf0b9f5b3bf4e8ddd1c6b12b59731a822 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:08:25 +0100 Subject: Try to fix test_gdb on s390x SLES 3.x Ignore empty lines in stderr. --- Lib/test/test_gdb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 8e3ccb0..ccef422 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -208,6 +208,8 @@ class DebuggerTests(unittest.TestCase): 'warning: ', ) for line in errlines: + if not line: + continue if not line.startswith(ignore_patterns): unexpected_errlines.append(line) -- cgit v0.12 From 0069aef51a176ec90fb93f4601636e8763e07c42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:15:55 +0100 Subject: Fix test_spwd on OpenIndiana Issue #18787: restore "bin" name in test_spwd but catch KeyError. --- Lib/test/test_spwd.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_spwd.py b/Lib/test/test_spwd.py index 3a11a2d..e893f3a 100644 --- a/Lib/test/test_spwd.py +++ b/Lib/test/test_spwd.py @@ -61,9 +61,14 @@ class TestSpwdRoot(unittest.TestCase): class TestSpwdNonRoot(unittest.TestCase): def test_getspnam_exception(self): - with self.assertRaises(PermissionError) as cm: - spwd.getspnam('root') - self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') + name = 'bin' + try: + with self.assertRaises(PermissionError) as cm: + spwd.getspnam(name) + except KeyError as exc: + self.skipTest("spwd entry %r doesn't exist: %s" % (name, exc)) + else: + self.assertEqual(str(cm.exception), '[Errno 13] Permission denied') if __name__ == "__main__": -- cgit v0.12 From cc73932125c9e1d6cada036d90873821ddda091a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 21:35:29 +0100 Subject: socketmodule.c: error if option larger than INT_MAX On Windows, socket.setsockopt() raises an OverflowError if the socket option is larger than INT_MAX bytes. --- Modules/socketmodule.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index ba3cefd..8df735d 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2458,13 +2458,26 @@ sock_setsockopt(PySocketSockObject *s, PyObject *args) if (!PyArg_ParseTuple(args, "iiy*:setsockopt", &level, &optname, &optval)) return NULL; +#ifdef MS_WINDOWS + if (optval.len > INT_MAX) { + PyBuffer_Release(&optval); + PyErr_Format(PyExc_OverflowError, + "socket option is larger than %i bytes", + INT_MAX); + return NULL; + } + res = setsockopt(s->sock_fd, level, optname, + optval.buf, (int)optval.len); +#else res = setsockopt(s->sock_fd, level, optname, optval.buf, optval.len); +#endif PyBuffer_Release(&optval); } - if (res < 0) + if (res < 0) { return s->errorhandler(); - Py_INCREF(Py_None); - return Py_None; + } + + Py_RETURN_NONE; } PyDoc_STRVAR(setsockopt_doc, -- cgit v0.12 From 5e14a38e8e84aa8a80651324c72c00f1d407e07c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Mar 2016 22:03:55 +0100 Subject: _tracemalloc: use compact key for traces Issue #26588: Optimize memory footprint of _tracemalloc before non-zero domain is used. Start with compact key (Py_uintptr_t) and also switch to pointer_t key when the first memory block with a non-zero domain is tracked. --- Modules/_tracemalloc.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 169fd2c..ca0ed3b 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -43,7 +43,7 @@ static struct { /* use domain in trace key? Variable protected by the GIL. */ int use_domain; -} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 1}; +} tracemalloc_config = {TRACEMALLOC_NOT_INITIALIZED, 0, 1, 0}; #if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) /* This lock is needed because tracemalloc_free() is called without @@ -519,6 +519,58 @@ traceback_new(void) } +static int +tracemalloc_use_domain_cb(_Py_hashtable_t *old_traces, + _Py_hashtable_entry_t *entry, void *user_data) +{ + Py_uintptr_t ptr; + pointer_t key; + _Py_hashtable_t *new_traces = (_Py_hashtable_t *)user_data; + const void *pdata = _Py_HASHTABLE_ENTRY_PDATA(old_traces, entry); + + _Py_HASHTABLE_ENTRY_READ_KEY(old_traces, entry, ptr); + key.ptr = ptr; + key.domain = DEFAULT_DOMAIN; + + return _Py_hashtable_set(new_traces, + sizeof(key), &key, + old_traces->data_size, pdata); +} + + +/* Convert tracemalloc_traces from compact key (Py_uintptr_t) to pointer_t key. + * Return 0 on success, -1 on error. */ +static int +tracemalloc_use_domain(void) +{ + _Py_hashtable_t *new_traces = NULL; + + assert(!tracemalloc_config.use_domain); + + new_traces = hashtable_new(sizeof(pointer_t), + sizeof(trace_t), + hashtable_hash_pointer_t, + hashtable_compare_pointer_t); + if (new_traces == NULL) { + return -1; + } + + if (_Py_hashtable_foreach(tracemalloc_traces, tracemalloc_use_domain_cb, + new_traces) < 0) + { + _Py_hashtable_destroy(new_traces); + return -1; + } + + _Py_hashtable_destroy(tracemalloc_traces); + tracemalloc_traces = new_traces; + + tracemalloc_config.use_domain = 1; + + return 0; +} + + static void tracemalloc_remove_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { @@ -563,6 +615,14 @@ tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr, return -1; } + if (!tracemalloc_config.use_domain && domain != DEFAULT_DOMAIN) { + /* first trace using a non-zero domain whereas traces use compact + (Py_uintptr_t) keys: switch to pointer_t keys. */ + if (tracemalloc_use_domain() < 0) { + return -1; + } + } + if (tracemalloc_config.use_domain) { entry = _Py_HASHTABLE_GET_ENTRY(tracemalloc_traces, key); } -- cgit v0.12 From 923590e397e255ae0f34a04eb662264de9b67f09 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 09:11:48 +0100 Subject: Fix DeprecationWarning on Windows Issue #25911: Use support.check_warnings() to expect or ignore DeprecationWarning in test_os. --- Lib/test/test_os.py | 116 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1c0dc5b..bac839d 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -66,6 +66,7 @@ except ImportError: from test.support.script_helper import assert_python_ok + root_in_posix = False if hasattr(os, 'geteuid'): root_in_posix = (os.geteuid() == 0) @@ -82,6 +83,23 @@ else: # Issue #14110: Some tests fail on FreeBSD if the user is in the wheel group. HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0 + +@contextlib.contextmanager +def ignore_deprecation_warnings(msg_regex, quiet=False): + with support.check_warnings((msg_regex, DeprecationWarning), quiet=quiet): + yield + + +@contextlib.contextmanager +def bytes_filename_warn(expected): + msg = 'The Windows bytes API has been deprecated' + if os.name == 'nt': + with ignore_deprecation_warnings(msg, quiet=not expected): + yield + else: + yield + + # Tests creating TESTFN class FileTests(unittest.TestCase): def setUp(self): @@ -305,8 +323,7 @@ class StatAttributeTests(unittest.TestCase): fname = self.fname.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: self.skipTest("cannot encode %a for the filesystem" % self.fname) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(True): self.check_stat_attributes(fname) def test_stat_result_pickle(self): @@ -443,15 +460,11 @@ class UtimeTests(unittest.TestCase): fp.write(b"ABC") def restore_float_times(state): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): os.stat_float_times(state) # ensure that st_atime and st_mtime are float - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - + with ignore_deprecation_warnings('stat_float_times'): old_float_times = os.stat_float_times(-1) self.addCleanup(restore_float_times, old_float_times) @@ -1024,8 +1037,7 @@ class BytesWalkTests(WalkTests): super().setUp() self.stack = contextlib.ExitStack() if os.name == 'nt': - self.stack.enter_context(warnings.catch_warnings()) - warnings.simplefilter("ignore", DeprecationWarning) + self.stack.enter_context(bytes_filename_warn(False)) def tearDown(self): self.stack.close() @@ -1580,8 +1592,7 @@ class LinkTests(unittest.TestCase): with open(file1, "w") as f1: f1.write("test") - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(False): os.link(file1, file2) with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) @@ -1873,10 +1884,12 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(support.TESTFN)), self.created_paths) + # bytes - self.assertEqual( - sorted(os.listdir(os.fsencode(support.TESTFN))), - [os.fsencode(path) for path in self.created_paths]) + with bytes_filename_warn(False): + self.assertEqual( + sorted(os.listdir(os.fsencode(support.TESTFN))), + [os.fsencode(path) for path in self.created_paths]) def test_listdir_extended_path(self): """Test when the path starts with '\\\\?\\'.""" @@ -1886,11 +1899,13 @@ class Win32ListdirTests(unittest.TestCase): self.assertEqual( sorted(os.listdir(path)), self.created_paths) + # bytes - path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) - self.assertEqual( - sorted(os.listdir(path)), - [os.fsencode(path) for path in self.created_paths]) + with bytes_filename_warn(False): + path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) + self.assertEqual( + sorted(os.listdir(path)), + [os.fsencode(path) for path in self.created_paths]) @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") @@ -1965,9 +1980,9 @@ class Win32SymlinkTests(unittest.TestCase): self.assertNotEqual(os.lstat(link), os.stat(link)) bytes_link = os.fsencode(link) - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) + with bytes_filename_warn(True): self.assertEqual(os.stat(bytes_link), os.stat(target)) + with bytes_filename_warn(True): self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) def test_12084(self): @@ -2529,36 +2544,37 @@ class Win32DeprecatedBytesAPI(unittest.TestCase): def test_deprecated(self): import nt filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - for func, *args in ( - (nt._getfullpathname, filename), - (nt._isdir, filename), - (os.access, filename, os.R_OK), - (os.chdir, filename), - (os.chmod, filename, 0o777), - (os.getcwdb,), - (os.link, filename, filename), - (os.listdir, filename), - (os.lstat, filename), - (os.mkdir, filename), - (os.open, filename, os.O_RDONLY), - (os.rename, filename, filename), - (os.rmdir, filename), - (os.startfile, filename), - (os.stat, filename), - (os.unlink, filename), - (os.utime, filename), - ): - self.assertRaises(DeprecationWarning, func, *args) + for func, *args in ( + (nt._getfullpathname, filename), + (nt._isdir, filename), + (os.access, filename, os.R_OK), + (os.chdir, filename), + (os.chmod, filename, 0o777), + (os.getcwdb,), + (os.link, filename, filename), + (os.listdir, filename), + (os.lstat, filename), + (os.mkdir, filename), + (os.open, filename, os.O_RDONLY), + (os.rename, filename, filename), + (os.rmdir, filename), + (os.startfile, filename), + (os.stat, filename), + (os.unlink, filename), + (os.utime, filename), + ): + with bytes_filename_warn(True): + try: + func(*args) + except OSError: + # ignore OSError, we only care about DeprecationWarning + pass @support.skip_unless_symlink def test_symlink(self): filename = os.fsencode(support.TESTFN) - with warnings.catch_warnings(): - warnings.simplefilter("error", DeprecationWarning) - self.assertRaises(DeprecationWarning, - os.symlink, filename, filename) + with bytes_filename_warn(True): + os.symlink(filename, filename) @unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size") @@ -2696,7 +2712,8 @@ class OSErrorTests(unittest.TestCase): for filenames, func, *func_args in funcs: for name in filenames: try: - func(name, *func_args) + with bytes_filename_warn(False): + func(name, *func_args) except OSError as err: self.assertIs(err.filename, name) else: @@ -3011,7 +3028,8 @@ class TestScandir(unittest.TestCase): def test_bytes(self): if os.name == "nt": # On Windows, os.scandir(bytes) must raise an exception - self.assertRaises(TypeError, os.scandir, b'.') + with bytes_filename_warn(True): + self.assertRaises(TypeError, os.scandir, b'.') return self.create_file("file.txt") -- cgit v0.12 From 5de16e80c14c7698992da8b08ae169e4be1d2ca3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 09:43:00 +0100 Subject: regrtest: fix --fromfile feature * Update code for the name regrtest output format. * Enhance also test_regrtest test on --fromfile --- Lib/test/libregrtest/main.py | 23 +++++++++++++++++------ Lib/test/test_regrtest.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index b954db5..de08f32 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -168,13 +168,21 @@ class Regrtest: if self.ns.fromfile: self.tests = [] + # regex to match 'test_builtin' in line: + # '0:00:00 [ 4/400] test_builtin -- test_dict took 1 sec' + regex = (r'^(?:[0-9]+:[0-9]+:[0-9]+ *)?' + r'(?:\[[0-9/ ]+\] *)?' + r'(test_[a-zA-Z0-9_]+)') + regex = re.compile(regex) with open(os.path.join(support.SAVEDCWD, self.ns.fromfile)) as fp: - count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') for line in fp: - line = count_pat.sub('', line) - guts = line.split() # assuming no test has whitespace in its name - if guts and not guts[0].startswith('#'): - self.tests.extend(guts) + line = line.strip() + if line.startswith('#'): + continue + match = regex.match(line) + if match is None: + continue + self.tests.append(match.group(1)) removepy(self.tests) @@ -194,7 +202,10 @@ class Regrtest: else: alltests = findtests(self.ns.testdir, stdtests, nottests) - self.selected = self.tests or self.ns.args or alltests + if not self.ns.fromfile: + self.selected = self.tests or self.ns.args or alltests + else: + self.selected = self.tests if self.ns.single: self.selected = self.selected[:1] try: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 03e7e1d..df8447c 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -628,6 +628,22 @@ class ArgsTestCase(BaseTestCase): # [2/2] test_2 filename = support.TESTFN self.addCleanup(support.unlink, filename) + + # test format '0:00:00 [2/7] test_opcodes -- test_grammar took 0 sec' + with open(filename, "w") as fp: + previous = None + for index, name in enumerate(tests, 1): + line = ("00:00:%02i [%s/%s] %s" + % (index, index, len(tests), name)) + if previous: + line += " -- %s took 0 sec" % previous + print(line, file=fp) + previous = name + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + + # test format '[2/7] test_opcodes' with open(filename, "w") as fp: for index, name in enumerate(tests, 1): print("[%s/%s] %s" % (index, len(tests), name), file=fp) @@ -635,6 +651,14 @@ class ArgsTestCase(BaseTestCase): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) + # test format 'test_opcodes' + with open(filename, "w") as fp: + for name in tests: + print(name, file=fp) + + output = self.run_tests('--fromfile', filename) + self.check_executed_tests(output, tests) + def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test("sigint", code=code) -- cgit v0.12 From 2b60b7237e81895b71e8763dd94946cd76bf2d16 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 11:55:29 +0100 Subject: regrtest: mention in tests run sequentially or in parallel --- Lib/test/libregrtest/main.py | 2 ++ Lib/test/libregrtest/runtest_mp.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index de08f32..c6d9ad0 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -305,6 +305,8 @@ class Regrtest: save_modules = sys.modules.keys() + print("Run tests sequentially") + previous_test = None for test_index, test in enumerate(self.tests, 1): start_time = time.monotonic() diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index e51b100..96db196 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -154,6 +154,8 @@ def run_tests_multiprocess(regrtest): workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] + print("Run tests in parallel using %s child processes" + % len(workers)) for worker in workers: worker.start() -- cgit v0.12 From 56db16cd445603f688084d9ace4c271b13e4ec01 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 12:04:15 +0100 Subject: regrtest: when parallel tests are interrupted, display progress --- Lib/test/libregrtest/runtest_mp.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index 96db196..aa97f05 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,6 +21,9 @@ from test.libregrtest.setup import setup_tests # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds +# If interrupted, display the wait process every N seconds +WAIT_PROGRESS = 2.0 # seconds + def run_test_in_subprocess(testname, ns): """Run the given test in a subprocess with --slaveargs. @@ -224,9 +227,18 @@ def run_tests_multiprocess(regrtest): if use_timeout: faulthandler.cancel_dump_traceback_later() - running = [worker.current_test for worker in workers] - running = list(filter(bool, running)) - if running: - print("Waiting for %s" % ', '.join(running)) - for worker in workers: - worker.join() + # If tests are interrupted, wait until tests complete + wait_start = time.monotonic() + while True: + running = [worker.current_test for worker in workers] + running = list(filter(bool, running)) + if not running: + break + + dt = time.monotonic() - wait_start + line = "Waiting for %s (%s tests)" % (', '.join(running), len(running)) + if dt >= WAIT_PROGRESS: + line = "%s since %.0f sec" % (line, dt) + print(line) + for worker in workers: + worker.join(WAIT_PROGRESS) -- cgit v0.12 From ba8b0a7db447fad7a7064e53424725b02ab5ccec Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 12:23:18 +0100 Subject: Enhance os._DummyDirEntry Issue #25911: * Try to fix test_os.BytesWalkTests on Windows * Try to mimick better the reference os.DirEntry on Windows * _DummyDirEntry now caches os.stat() result * _DummyDirEntry constructor now tries to get os.stat() --- Lib/os.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index 0ea7b62..c3ce05d 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -439,15 +439,36 @@ def walk(top, topdown=True, onerror=None, followlinks=False): yield top, dirs, nondirs class _DummyDirEntry: + """Dummy implementation of DirEntry + + Only used internally by os.walk(bytes). Since os.walk() doesn't need the + follow_symlinks parameter: don't implement it, always follow symbolic + links. + """ + def __init__(self, dir, name): self.name = name self.path = path.join(dir, name) + # Mimick FindFirstFile/FindNextFile: we should get file attributes + # while iterating on a directory + self._stat = None + try: + self.stat() + except OSError: + pass + + def stat(self): + if self._stat is None: + self._stat = stat(self.path) + return self._stat def is_dir(self): - return path.isdir(self.path) + stat = self.stat() + return st.S_ISDIR(stat.st_mode) def is_symlink(self): - return path.islink(self.path) + stat = self.stat() + return st.S_ISLNK(stat.st_mode) class _dummy_scandir: # listdir-based implementation for bytes patches on Windows -- cgit v0.12 From e321274b2eeaf9cfaccbc8d40c4056ab72040e1b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 13:44:19 +0100 Subject: Enhance and modernize test_genericpath * Replace "try/finally: os.remove()" with self.addCleanup(support.unlink) or self.addCleanup(support.rmdir): the support function handles the case when the file doesn't exist * Replace "try/finally: f.close()" with "with open(...) as f:" * test_getsize: add a second test with a different size * Create file using "x" mode to ensure that the file didn't exist before, to detect bugs in tests * Open files in unbuffered mode (buferring=0) to write immediatly data on disk * Replace map() with simpler code * Split isdir() unit test into two units tests to make them less dependant, same change for isfile() test * test_samefile(): test also two different files --- Lib/test/test_genericpath.py | 270 +++++++++++++++++++++---------------------- 1 file changed, 134 insertions(+), 136 deletions(-) diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 86fc2de..9c28a68 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -10,11 +10,9 @@ import warnings from test import support -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass +def create_file(filename, data=b'foo'): + with open(filename, 'xb', 0) as fp: + fp.write(data) class GenericTest: @@ -97,52 +95,47 @@ class GenericTest: self.assertNotEqual(s1[n:n+1], s2[n:n+1]) def test_getsize(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertEqual(self.pathmodule.getsize(support.TESTFN), 3) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) - def test_time(self): - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - f = open(support.TESTFN, "ab") + create_file(filename, b'Hello') + self.assertEqual(self.pathmodule.getsize(filename), 5) + os.remove(filename) + + create_file(filename, b'Hello World!') + self.assertEqual(self.pathmodule.getsize(filename), 12) + + def test_filetime(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + create_file(filename, b'foo') + + with open(filename, "ab", 0) as f: f.write(b"bar") - f.close() - f = open(support.TESTFN, "rb") - d = f.read() - f.close() - self.assertEqual(d, b"foobar") - - self.assertLessEqual( - self.pathmodule.getctime(support.TESTFN), - self.pathmodule.getmtime(support.TESTFN) - ) - finally: - if not f.closed: - f.close() - support.unlink(support.TESTFN) + + with open(filename, "rb", 0) as f: + data = f.read() + self.assertEqual(data, b"foobar") + + self.assertLessEqual( + self.pathmodule.getctime(filename), + self.pathmodule.getmtime(filename) + ) def test_exists(self): - self.assertIs(self.pathmodule.exists(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + self.assertIs(self.pathmodule.exists(filename), False) + + with open(filename, "xb") as f: f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.exists(support.TESTFN), True) - if not self.pathmodule == genericpath: - self.assertIs(self.pathmodule.lexists(support.TESTFN), - True) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) + + self.assertIs(self.pathmodule.exists(filename), True) + + if not self.pathmodule == genericpath: + self.assertIs(self.pathmodule.lexists(filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_exists_fd(self): @@ -154,53 +147,66 @@ class GenericTest: os.close(w) self.assertFalse(self.pathmodule.exists(r)) - def test_isdir(self): - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isdir(support.TESTFN), False) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isdir(support.TESTFN), True) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) - - def test_isfile(self): - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - f = open(support.TESTFN, "wb") - try: - f.write(b"foo") - f.close() - self.assertIs(self.pathmodule.isfile(support.TESTFN), True) - os.remove(support.TESTFN) - os.mkdir(support.TESTFN) - self.assertIs(self.pathmodule.isfile(support.TESTFN), False) - os.rmdir(support.TESTFN) - finally: - if not f.close(): - f.close() - support.unlink(support.TESTFN) - safe_rmdir(support.TESTFN) + def test_isdir_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + create_file(filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + def test_isdir_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isdir(filename), False) + + os.mkdir(filename) + self.assertIs(self.pathmodule.isdir(filename), True) + + def test_isfile_file(self): + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + self.assertIs(self.pathmodule.isfile(filename), False) - @staticmethod - def _create_file(filename): - with open(filename, 'wb') as f: - f.write(b'foo') + create_file(filename) + self.assertIs(self.pathmodule.isfile(filename), True) + + def test_isfile_dir(self): + filename = support.TESTFN + self.addCleanup(support.rmdir, filename) + self.assertIs(self.pathmodule.isfile(filename), False) + + os.mkdir(filename) + self.assertIs(self.pathmodule.isfile(filename), False) def test_samefile(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - self.assertTrue(self.pathmodule.samefile(test_fn, test_fn)) - self.assertRaises(TypeError, self.pathmodule.samefile) - finally: - os.remove(test_fn) + file1 = support.TESTFN + file2 = support.TESTFN + "2" + self.addCleanup(support.unlink, file1) + self.addCleanup(support.unlink, file2) + + create_file(file1) + self.assertTrue(self.pathmodule.samefile(file1, file1)) + + create_file(file2) + self.assertFalse(self.pathmodule.samefile(file1, file2)) + + self.assertRaises(TypeError, self.pathmodule.samefile) + + def _test_samefile_on_link_func(self, func): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) @support.skip_unless_symlink def test_samefile_on_symlink(self): @@ -209,31 +215,37 @@ class GenericTest: def test_samefile_on_link(self): self._test_samefile_on_link_func(os.link) - def _test_samefile_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) + def test_samestat(self): + test_fn1 = support.TESTFN + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) - func(test_fn1, test_fn2) - self.assertTrue(self.pathmodule.samefile(test_fn1, test_fn2)) - os.remove(test_fn2) + create_file(test_fn1) + stat1 = os.stat(test_fn1) + self.assertTrue(self.pathmodule.samestat(stat1, os.stat(test_fn1))) - self._create_file(test_fn2) - self.assertFalse(self.pathmodule.samefile(test_fn1, test_fn2)) - finally: - os.remove(test_fn1) - os.remove(test_fn2) + create_file(test_fn2) + stat2 = os.stat(test_fn2) + self.assertFalse(self.pathmodule.samestat(stat1, stat2)) - def test_samestat(self): - try: - test_fn = support.TESTFN + "1" - self._create_file(test_fn) - test_fns = [test_fn]*2 - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - finally: - os.remove(test_fn) + self.assertRaises(TypeError, self.pathmodule.samestat) + + def _test_samestat_on_link_func(self, func): + test_fn1 = support.TESTFN + "1" + test_fn2 = support.TESTFN + "2" + self.addCleanup(support.unlink, test_fn1) + self.addCleanup(support.unlink, test_fn2) + + create_file(test_fn1) + func(test_fn1, test_fn2) + self.assertTrue(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) + os.remove(test_fn2) + + create_file(test_fn2) + self.assertFalse(self.pathmodule.samestat(os.stat(test_fn1), + os.stat(test_fn2))) @support.skip_unless_symlink def test_samestat_on_symlink(self): @@ -242,31 +254,17 @@ class GenericTest: def test_samestat_on_link(self): self._test_samestat_on_link_func(os.link) - def _test_samestat_on_link_func(self, func): - try: - test_fn1 = support.TESTFN + "1" - test_fn2 = support.TESTFN + "2" - self._create_file(test_fn1) - test_fns = (test_fn1, test_fn2) - func(*test_fns) - stats = map(os.stat, test_fns) - self.assertTrue(self.pathmodule.samestat(*stats)) - os.remove(test_fn2) - - self._create_file(test_fn2) - stats = map(os.stat, test_fns) - self.assertFalse(self.pathmodule.samestat(*stats)) - - self.assertRaises(TypeError, self.pathmodule.samestat) - finally: - os.remove(test_fn1) - os.remove(test_fn2) - def test_sameopenfile(self): - fname = support.TESTFN + "1" - with open(fname, "wb") as a, open(fname, "wb") as b: - self.assertTrue(self.pathmodule.sameopenfile( - a.fileno(), b.fileno())) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + create_file(filename) + + with open(filename, "rb", 0) as fp1: + fd1 = fp1.fileno() + with open(filename, "rb", 0) as fp2: + fd2 = fp2.fileno() + self.assertTrue(self.pathmodule.sameopenfile(fd1, fd2)) + class TestGenericTest(GenericTest, unittest.TestCase): # Issue 16852: GenericTest can't inherit from unittest.TestCase -- cgit v0.12 From bc6b72ed06318a80a68b4bbf513a0e055b102e11 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 13:55:58 +0100 Subject: Closes #26620: Fix ResourceWarning in test_urllib2_localnet * Use context manager on urllib objects to ensure that they are closed on error * Use self.addCleanup() to cleanup resources even if a test is interrupted with CTRL+c --- Lib/test/test_urllib2_localnet.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index c8b37ee..e9564fd 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -289,12 +289,12 @@ class BasicAuthTests(unittest.TestCase): def http_server_with_basic_auth_handler(*args, **kwargs): return BasicAuthHandler(*args, **kwargs) self.server = LoopbackHttpServerThread(http_server_with_basic_auth_handler) + self.addCleanup(self.server.stop) self.server_url = 'http://127.0.0.1:%s' % self.server.port self.server.start() self.server.ready.wait() def tearDown(self): - self.server.stop() super(BasicAuthTests, self).tearDown() def test_basic_auth_success(self): @@ -438,17 +438,13 @@ class TestUrlopen(unittest.TestCase): def setUp(self): super(TestUrlopen, self).setUp() + # Ignore proxies for localhost tests. - self.old_environ = os.environ.copy() + def restore_environ(old_environ): + os.environ.clear() + os.environ.update(old_environ) + self.addCleanup(restore_environ, os.environ.copy()) os.environ['NO_PROXY'] = '*' - self.server = None - - def tearDown(self): - if self.server is not None: - self.server.stop() - os.environ.clear() - os.environ.update(self.old_environ) - super(TestUrlopen, self).tearDown() def urlopen(self, url, data=None, **kwargs): l = [] @@ -469,6 +465,7 @@ class TestUrlopen(unittest.TestCase): handler = GetRequestHandler(responses) self.server = LoopbackHttpServerThread(handler) + self.addCleanup(self.server.stop) self.server.start() self.server.ready.wait() port = self.server.port @@ -592,7 +589,8 @@ class TestUrlopen(unittest.TestCase): handler = self.start_server() req = urllib.request.Request("http://localhost:%s/" % handler.port, headers={"Range": "bytes=20-39"}) - urllib.request.urlopen(req) + with urllib.request.urlopen(req): + pass self.assertEqual(handler.headers_received["Range"], "bytes=20-39") def test_basic(self): @@ -608,22 +606,21 @@ class TestUrlopen(unittest.TestCase): def test_info(self): handler = self.start_server() - try: - open_url = urllib.request.urlopen( - "http://localhost:%s" % handler.port) + open_url = urllib.request.urlopen( + "http://localhost:%s" % handler.port) + with open_url: info_obj = open_url.info() - self.assertIsInstance(info_obj, email.message.Message, - "object returned by 'info' is not an " - "instance of email.message.Message") - self.assertEqual(info_obj.get_content_subtype(), "plain") - finally: - self.server.stop() + self.assertIsInstance(info_obj, email.message.Message, + "object returned by 'info' is not an " + "instance of email.message.Message") + self.assertEqual(info_obj.get_content_subtype(), "plain") def test_geturl(self): # Make sure same URL as opened is returned by geturl. handler = self.start_server() open_url = urllib.request.urlopen("http://localhost:%s" % handler.port) - url = open_url.geturl() + with open_url: + url = open_url.geturl() self.assertEqual(url, "http://localhost:%s" % handler.port) def test_iteration(self): -- cgit v0.12 From f95a19b900262c32d1038a6ab9a6808a58656c34 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 16:50:41 +0100 Subject: test_os: use @support.requires_linux_version --- Lib/test/test_os.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bac839d..b317e33 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -15,7 +15,6 @@ import locale import mmap import os import pickle -import platform import re import shutil import signal @@ -2456,14 +2455,14 @@ def supports_extended_attributes(): return False finally: support.unlink(support.TESTFN) - # Kernels < 2.6.39 don't respect setxattr flags. - kernel_version = platform.release() - m = re.match("2.6.(\d{1,2})", kernel_version) - return m is None or int(m.group(1)) >= 39 + + return True @unittest.skipUnless(supports_extended_attributes(), "no non-broken extended attribute support") +# Kernels < 2.6.39 don't respect setxattr flags. +@support.requires_linux_version(2, 6, 39) class ExtendedAttributeTests(unittest.TestCase): def tearDown(self): -- cgit v0.12 From ae39d236b41ad369215dc523b63000f4860517cc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:12:55 +0100 Subject: Enhance and modernize test_os * add create_file() helper function * create files using "x" mode instead of "w" to detect when a previous test forget to remove a file * open file for writing in unbuferred mode (buffering=0) * replace "try/finally: unlink" with self.addCleanup(support.unlink) * register unlink cleanup function *before* creating new files --- Lib/test/test_os.py | 153 ++++++++++++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 77 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b317e33..17484fe 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -99,6 +99,11 @@ def bytes_filename_warn(expected): yield +def create_file(filename, content=b'content'): + with open(filename, "xb", 0) as fp: + fp.write(content) + + # Tests creating TESTFN class FileTests(unittest.TestCase): def setUp(self): @@ -157,9 +162,8 @@ class FileTests(unittest.TestCase): "needs INT_MAX < PY_SSIZE_T_MAX") @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) def test_large_read(self, size): - with open(support.TESTFN, "wb") as fp: - fp.write(b'test') self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b'test') # Issue #21932: Make sure that os.read() does not raise an # OverflowError for size larger than INT_MAX @@ -216,11 +220,12 @@ class FileTests(unittest.TestCase): def test_replace(self): TESTFN2 = support.TESTFN + ".2" - with open(support.TESTFN, 'w') as f: - f.write("1") - with open(TESTFN2, 'w') as f: - f.write("2") - self.addCleanup(os.unlink, TESTFN2) + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(support.unlink, TESTFN2) + + create_file(support.TESTFN, b"1") + create_file(TESTFN2, b"2") + os.replace(support.TESTFN, TESTFN2) self.assertRaises(FileNotFoundError, os.stat, support.TESTFN) with open(TESTFN2, 'r') as f: @@ -245,8 +250,7 @@ class StatAttributeTests(unittest.TestCase): def setUp(self): self.fname = support.TESTFN self.addCleanup(support.unlink, self.fname) - with open(self.fname, 'wb') as fp: - fp.write(b"ABC") + create_file(self.fname, b"ABC") @unittest.skipUnless(hasattr(os, 'stat'), 'test needs os.stat()') def check_stat_attributes(self, fname): @@ -455,8 +459,7 @@ class UtimeTests(unittest.TestCase): self.addCleanup(support.rmtree, self.dirname) os.mkdir(self.dirname) - with open(self.fname, 'wb') as fp: - fp.write(b"ABC") + create_file(self.fname) def restore_float_times(state): with ignore_deprecation_warnings('stat_float_times'): @@ -555,7 +558,7 @@ class UtimeTests(unittest.TestCase): "fd support for utime required for this test.") def test_utime_fd(self): def set_time(filename, ns): - with open(filename, 'wb') as fp: + with open(filename, 'wb', 0) as fp: # use a file descriptor to test futimens(timespec) # or futimes(timeval) os.utime(fp.fileno(), ns=ns) @@ -1213,8 +1216,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dira, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dira, 'file.txt')) os.removedirs(dirb) self.assertFalse(os.path.exists(dirb)) self.assertTrue(os.path.exists(dira)) @@ -1225,8 +1227,7 @@ class RemoveDirsTests(unittest.TestCase): os.mkdir(dira) dirb = os.path.join(dira, 'dirb') os.mkdir(dirb) - with open(os.path.join(dirb, 'file.txt'), 'w') as f: - f.write('text') + create_file(os.path.join(dirb, 'file.txt')) with self.assertRaises(OSError): os.removedirs(dirb) self.assertTrue(os.path.exists(dirb)) @@ -1236,7 +1237,7 @@ class RemoveDirsTests(unittest.TestCase): class DevNullTests(unittest.TestCase): def test_devnull(self): - with open(os.devnull, 'wb') as f: + with open(os.devnull, 'wb', 0) as f: f.write(b'hello') f.close() with open(os.devnull, 'rb') as f: @@ -1323,9 +1324,9 @@ class URandomFDTests(unittest.TestCase): def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. - with open(support.TESTFN, 'wb') as f: - f.write(b"x" * 256) - self.addCleanup(os.unlink, support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b"x" * 256) + code = """if 1: import os import sys @@ -1464,12 +1465,10 @@ class Win32ErrorTests(unittest.TestCase): self.assertRaises(OSError, os.chdir, support.TESTFN) def test_mkdir(self): - f = open(support.TESTFN, "w") - try: + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, "w") as f: self.assertRaises(OSError, os.mkdir, support.TESTFN) - finally: - f.close() - os.unlink(support.TESTFN) def test_utime(self): self.assertRaises(OSError, os.utime, support.TESTFN, None) @@ -1988,42 +1987,36 @@ class Win32SymlinkTests(unittest.TestCase): level1 = os.path.abspath(support.TESTFN) level2 = os.path.join(level1, "level2") level3 = os.path.join(level2, "level3") - try: - os.mkdir(level1) - os.mkdir(level2) - os.mkdir(level3) + self.addCleanup(support.rmtree, level1) - file1 = os.path.abspath(os.path.join(level1, "file1")) + os.mkdir(level1) + os.mkdir(level2) + os.mkdir(level3) - with open(file1, "w") as f: - f.write("file1") + file1 = os.path.abspath(os.path.join(level1, "file1")) + create_file(file1) - orig_dir = os.getcwd() - try: - os.chdir(level2) - link = os.path.join(level2, "link") - os.symlink(os.path.relpath(file1), "link") - self.assertIn("link", os.listdir(os.getcwd())) - - # Check os.stat calls from the same dir as the link - self.assertEqual(os.stat(file1), os.stat("link")) - - # Check os.stat calls from a dir below the link - os.chdir(level1) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - - # Check os.stat calls from a dir above the link - os.chdir(level3) - self.assertEqual(os.stat(file1), - os.stat(os.path.relpath(link))) - finally: - os.chdir(orig_dir) - except OSError as err: - self.fail(err) + orig_dir = os.getcwd() + try: + os.chdir(level2) + link = os.path.join(level2, "link") + os.symlink(os.path.relpath(file1), "link") + self.assertIn("link", os.listdir(os.getcwd())) + + # Check os.stat calls from the same dir as the link + self.assertEqual(os.stat(file1), os.stat("link")) + + # Check os.stat calls from a dir below the link + os.chdir(level1) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + + # Check os.stat calls from a dir above the link + os.chdir(level3) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) finally: - os.remove(file1) - shutil.rmtree(level1) + os.chdir(orig_dir) @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") @@ -2159,8 +2152,8 @@ class ProgramPriorityTests(unittest.TestCase): try: new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) if base >= 19 and new_prio <= 19: - raise unittest.SkipTest( - "unable to reliably test setpriority at current nice level of %s" % base) + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) else: self.assertEqual(new_prio, base + 1) finally: @@ -2270,8 +2263,7 @@ class TestSendfile(unittest.TestCase): @classmethod def setUpClass(cls): cls.key = support.threading_setup() - with open(support.TESTFN, "wb") as f: - f.write(cls.DATA) + create_file(support.TESTFN, cls.DATA) @classmethod def tearDownClass(cls): @@ -2421,10 +2413,11 @@ class TestSendfile(unittest.TestCase): def test_trailers(self): TESTFN2 = support.TESTFN + "2" file_data = b"abcdef" - with open(TESTFN2, 'wb') as f: - f.write(file_data) - with open(TESTFN2, 'rb')as f: - self.addCleanup(os.remove, TESTFN2) + + self.addCleanup(support.unlink, TESTFN2) + create_file(TESTFN2, file_data) + + with open(TESTFN2, 'rb') as f: os.sendfile(self.sockno, f.fileno(), 0, len(file_data), trailers=[b"1234"]) self.client.close() @@ -2447,8 +2440,9 @@ class TestSendfile(unittest.TestCase): def supports_extended_attributes(): if not hasattr(os, "setxattr"): return False + try: - with open(support.TESTFN, "wb") as fp: + with open(support.TESTFN, "xb", 0) as fp: try: os.setxattr(fp.fileno(), b"user.test", b"") except OSError: @@ -2465,17 +2459,18 @@ def supports_extended_attributes(): @support.requires_linux_version(2, 6, 39) class ExtendedAttributeTests(unittest.TestCase): - def tearDown(self): - support.unlink(support.TESTFN) - def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs): fn = support.TESTFN - open(fn, "wb").close() + self.addCleanup(support.unlink, fn) + create_file(fn) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + init_xattr = listxattr(fn) self.assertIsInstance(init_xattr, list) + setxattr(fn, s("user.test"), b"", **kwargs) xattr = set(init_xattr) xattr.add("user.test") @@ -2483,19 +2478,24 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"") setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE, **kwargs) self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"hello") + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE, **kwargs) self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(OSError) as cm: setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE, **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE, **kwargs) xattr.add("user.test2") self.assertEqual(set(listxattr(fn)), xattr) removexattr(fn, s("user.test"), **kwargs) + with self.assertRaises(OSError) as cm: getxattr(fn, s("user.test"), **kwargs) self.assertEqual(cm.exception.errno, errno.ENODATA) + xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") @@ -2508,11 +2508,11 @@ class ExtendedAttributeTests(unittest.TestCase): self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) def _check_xattrs(self, *args, **kwargs): - def make_bytes(s): - return bytes(s, "ascii") self._check_xattrs_str(str, *args, **kwargs) support.unlink(support.TESTFN) - self._check_xattrs_str(make_bytes, *args, **kwargs) + + self._check_xattrs_str(os.fsencode, *args, **kwargs) + support.unlink(support.TESTFN) def test_simple(self): self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, @@ -2527,10 +2527,10 @@ class ExtendedAttributeTests(unittest.TestCase): with open(path, "rb") as fp: return os.getxattr(fp.fileno(), *args) def setxattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.setxattr(fp.fileno(), *args) def removexattr(path, *args): - with open(path, "wb") as fp: + with open(path, "wb", 0) as fp: os.removexattr(fp.fileno(), *args) def listxattr(path, *args): with open(path, "rb") as fp: @@ -2844,8 +2844,7 @@ class TestScandir(unittest.TestCase): def create_file(self, name="file.txt"): filename = os.path.join(self.path, name) - with open(filename, "wb") as fp: - fp.write(b'python') + create_file(filename, b'python') return filename def get_entries(self, names): -- cgit v0.12 From 3899b549e3d1acb1f0f7bc5d226cc3632675d36d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:21:17 +0100 Subject: test_os: use support.rmtree() to cleanup WalkTests --- Lib/test/test_os.py | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 17484fe..bc24e47 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -812,6 +812,7 @@ class WalkTests(unittest.TestCase): def setUp(self): join = os.path.join + self.addCleanup(support.rmtree, support.TESTFN) # Build: # TESTFN/ @@ -922,22 +923,6 @@ class WalkTests(unittest.TestCase): else: self.fail("Didn't follow symlink with followlinks=True") - def tearDown(self): - # Tear everything down. This is a decent use for bottom-up on - # Windows, which doesn't have a recursive delete command. The - # (not so) subtlety is that rmdir will fail unless the dir's - # kids are removed first, so bottom up is essential. - for root, dirs, files in os.walk(support.TESTFN, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - dirname = os.path.join(root, name) - if not os.path.islink(dirname): - os.rmdir(dirname) - else: - os.remove(dirname) - os.rmdir(support.TESTFN) - def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1020,19 +1005,6 @@ class FwalkTests(WalkTests): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) - def tearDown(self): - # cleanup - for root, dirs, files, rootfd in os.fwalk(support.TESTFN, topdown=False): - for name in files: - os.unlink(name, dir_fd=rootfd) - for name in dirs: - st = os.stat(name, dir_fd=rootfd, follow_symlinks=False) - if stat.S_ISDIR(st.st_mode): - os.rmdir(name, dir_fd=rootfd) - else: - os.unlink(name, dir_fd=rootfd) - os.rmdir(support.TESTFN) - class BytesWalkTests(WalkTests): """Tests for os.walk() with bytes.""" def setUp(self): -- cgit v0.12 From e40390473dc1303c550664457f5b7b2582592086 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:42:10 +0100 Subject: support.temp_dir(): call support.rmtree() instead of shutil.rmtree() --- Lib/test/support/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index a2ef93d..7914943 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -902,7 +902,7 @@ def temp_dir(path=None, quiet=False): yield path finally: if dir_created: - shutil.rmtree(path) + rmtree(path) @contextlib.contextmanager def change_cwd(path, quiet=False): -- cgit v0.12 From 4ffcc3ee1e982b1c739a3c1d6fb948960b31c3dd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:43:53 +0100 Subject: Cleanup regrtest.py * Move code into a new _main() function * Fix loop to cleanup sys.path * Remove unused import --- Lib/test/regrtest.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index fcc3937..9cbb926 100644 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,21 +11,28 @@ import importlib import os import sys -from test.libregrtest import main, main_in_temp_cwd +from test.libregrtest import main_in_temp_cwd -if __name__ == '__main__': +# alias needed by other scripts +main = main_in_temp_cwd + + +def _main(): + global __file__ + # Remove regrtest.py's own directory from the module search path. Despite # the elimination of implicit relative imports, this is still needed to # ensure that submodules of the test package do not inappropriately appear # as top-level modules even when people (or buildbots!) invoke regrtest.py # directly instead of using the -m switch mydir = os.path.abspath(os.path.normpath(os.path.dirname(sys.argv[0]))) - i = len(sys.path) + i = len(sys.path) - 1 while i >= 0: - i -= 1 if os.path.abspath(os.path.normpath(sys.path[i])) == mydir: del sys.path[i] + else: + i -= 1 # findtestdir() gets the dirname out of __file__, so we have to make it # absolute before changing the working directory. @@ -36,4 +43,8 @@ if __name__ == '__main__': # sanity check assert __file__ == os.path.abspath(sys.argv[0]) - main_in_temp_cwd() + main() + + +if __name__ == '__main__': + _main() -- cgit v0.12 From 8c08e0db8fa47c0c8780607a0d803d4964fdffa6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:46:24 +0100 Subject: rt.bat: use -m test instead of Lib\test\regrtest.py --- PCbuild/rt.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index 2d93b80..7d4d071 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -42,7 +42,7 @@ if "%1"=="-x64" (set prefix=%pcbuild%amd64\) & shift & goto CheckOpts if NOT "%1"=="" (set regrtestargs=%regrtestargs% %1) & shift & goto CheckOpts set exe=%prefix%python%suffix%.exe -set cmd="%exe%" %dashO% -Wd -E -bb "%pcbuild%..\lib\test\regrtest.py" %regrtestargs% +set cmd="%exe%" %dashO% -Wd -E -bb -m test %regrtestargs% if defined qmode goto Qmode echo Deleting .pyc/.pyo files ... -- cgit v0.12 From 3aac0adfe01b30fc58c638c5ab61844e80b3fd66 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Mar 2016 17:53:20 +0100 Subject: Cleanup regrtest "main()" function * Rename libregrtest.main_in_temp_cwd() to libregrtest.main() * Add regrtest.main_in_temp_cwd() alias to libregrtest.main() * Move old main_in_temp_cwd() code into libregrtest.Regrtest.main() * Update multiple scripts to call libregrtest.main() --- Lib/test/__main__.py | 5 ++--- Lib/test/autotest.py | 5 ++--- Lib/test/libregrtest/__init__.py | 5 ++++- Lib/test/libregrtest/main.py | 45 +++++++++++++++++++------------------ Lib/test/regrtest.py | 6 ++--- Lib/test/test_importlib/regrtest.py | 4 ++-- PC/testpy.py | 4 ++-- 7 files changed, 38 insertions(+), 36 deletions(-) diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index d5fbe15..19a6b2b 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,3 +1,2 @@ -from test import regrtest - -regrtest.main_in_temp_cwd() +from test.libregrtest import main +main() diff --git a/Lib/test/autotest.py b/Lib/test/autotest.py index 41c2088..fa85cc1 100644 --- a/Lib/test/autotest.py +++ b/Lib/test/autotest.py @@ -1,6 +1,5 @@ # This should be equivalent to running regrtest.py from the cmdline. # It can be especially handy if you're in an interactive shell, e.g., # from test import autotest. - -from test import regrtest -regrtest.main() +from test.libregrtest import main +main() diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py index 9f7b1c1..7ba0e6e 100644 --- a/Lib/test/libregrtest/__init__.py +++ b/Lib/test/libregrtest/__init__.py @@ -1,2 +1,5 @@ +# We import importlib *ASAP* in order to test #15386 +import importlib + from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES -from test.libregrtest.main import main, main_in_temp_cwd +from test.libregrtest.main import main diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index c6d9ad0..e1367da 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -415,6 +415,28 @@ class Regrtest: os.system("leaks %d" % os.getpid()) def main(self, tests=None, **kwargs): + global TEMPDIR + + if sysconfig.is_python_build(): + try: + os.mkdir(TEMPDIR) + except FileExistsError: + pass + + # Define a writable temp dir that will be used as cwd while running + # the tests. The name of the dir includes the pid to allow parallel + # testing (see the -j option). + test_cwd = 'test_python_{}'.format(os.getpid()) + test_cwd = os.path.join(TEMPDIR, test_cwd) + + # Run the tests in a context manager that temporarily changes the CWD to a + # temporary and writable directory. If it's not possible to create or + # change the CWD, the original CWD will be used. The original CWD is + # available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + self._main(tests, kwargs) + + def _main(self, tests, kwargs): self.ns = self.parse_args(kwargs) if self.ns.slaveargs is not None: @@ -473,26 +495,5 @@ def printlist(x, width=70, indent=4): def main(tests=None, **kwargs): + """Run the Python suite.""" Regrtest().main(tests=tests, **kwargs) - - -def main_in_temp_cwd(): - """Run main() in a temporary working directory.""" - if sysconfig.is_python_build(): - try: - os.mkdir(TEMPDIR) - except FileExistsError: - pass - - # Define a writable temp dir that will be used as cwd while running - # the tests. The name of the dir includes the pid to allow parallel - # testing (see the -j option). - test_cwd = 'test_python_{}'.format(os.getpid()) - test_cwd = os.path.join(TEMPDIR, test_cwd) - - # Run the tests in a context manager that temporarily changes the CWD to a - # temporary and writable directory. If it's not possible to create or - # change the CWD, the original CWD will be used. The original CWD is - # available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - main() diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 9cbb926..21b0edf 100644 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -11,11 +11,11 @@ import importlib import os import sys -from test.libregrtest import main_in_temp_cwd +from test.libregrtest import main -# alias needed by other scripts -main = main_in_temp_cwd +# Alias for backward compatibility (just in case) +main_in_temp_cwd = main def _main(): diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py index a5be11f..98c815c 100644 --- a/Lib/test/test_importlib/regrtest.py +++ b/Lib/test/test_importlib/regrtest.py @@ -8,10 +8,10 @@ this script. """ import importlib import sys -from test import regrtest +from test import libregrtest if __name__ == '__main__': __builtins__.__import__ = importlib.__import__ sys.path_importer_cache.clear() - regrtest.main(quiet=True, verbose2=True) + libregrtest.main(quiet=True, verbose2=True) diff --git a/PC/testpy.py b/PC/testpy.py index 4ef3d4f..709f35c 100644 --- a/PC/testpy.py +++ b/PC/testpy.py @@ -26,5 +26,5 @@ for dir in sys.path: # Add the "test" directory to PYTHONPATH. sys.path = sys.path + [test] -import regrtest # Standard Python tester. -regrtest.main() +import libregrtest # Standard Python tester. +libregrtest.main() -- cgit v0.12 From 4f17426437a5bdf458f8628936c3e37909fca448 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 00:40:59 +0100 Subject: Fix bug in __import__ during Python shutdown Issue #26637: The importlib module now emits an ImportError rather than a TypeError if __import__() is tried during the Python shutdown process but sys.path is already cleared (set to None). --- Lib/importlib/_bootstrap.py | 11 +- Misc/NEWS | 5 + Python/importlib.h | 884 ++++++++++++++++++++++---------------------- 3 files changed, 459 insertions(+), 441 deletions(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 880c493..fa99f56 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -878,13 +878,20 @@ def _find_spec_legacy(finder, name, path): def _find_spec(name, path, target=None): """Find a module's loader.""" - if sys.meta_path is not None and not sys.meta_path: + meta_path = sys.meta_path + if meta_path is None: + # PyImport_Cleanup() is running or has been called. + raise ImportError("sys.meta_path is None, Python is likely " + "shutting down") + + if not meta_path: _warnings.warn('sys.meta_path is empty', ImportWarning) + # We check sys.modules here for the reload case. While a passed-in # target will usually indicate a reload there is no guarantee, whereas # sys.modules provides one. is_reload = name in sys.modules - for finder in sys.meta_path: + for finder in meta_path: with _ImportLockContext(): try: find_spec = finder.find_spec diff --git a/Misc/NEWS b/Misc/NEWS index f483735..d684c82 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,11 @@ Core and Builtins Library ------- +- Issue #26637: The :mod:`importlib` module now emits an :exc:`ImportError` + rather than a :exc:`TypeError` if :func:`__import__` is tried during the + Python shutdown process but :data:`sys.path` is already cleared (set to + ``None``). + - Issue #21925: :func:`warnings.formatwarning` now catches exceptions when calling :func;`linecache.getline` and :func:`tracemalloc.get_object_traceback` to be able to log diff --git a/Python/importlib.h b/Python/importlib.h index d983114..685c51a 100644 --- a/Python/importlib.h +++ b/Python/importlib.h @@ -1573,443 +1573,449 @@ const unsigned char _Py_M__importlib[] = { 11,0,0,0,218,17,95,102,105,110,100,95,115,112,101,99, 95,108,101,103,97,99,121,102,3,0,0,115,8,0,0,0, 0,3,18,1,12,1,4,1,114,174,0,0,0,99,3,0, - 0,0,0,0,0,0,9,0,0,0,27,0,0,0,67,0, - 0,0,115,42,1,0,0,116,0,0,106,1,0,100,1,0, - 107,9,0,114,41,0,116,0,0,106,1,0,12,114,41,0, - 116,2,0,106,3,0,100,2,0,116,4,0,131,2,0,1, - 124,0,0,116,0,0,106,5,0,107,6,0,125,3,0,120, - 235,0,116,0,0,106,1,0,68,93,220,0,125,4,0,116, - 6,0,131,0,0,143,90,0,1,121,13,0,124,4,0,106, - 7,0,125,5,0,87,110,51,0,4,116,8,0,107,10,0, - 114,148,0,1,1,1,116,9,0,124,4,0,124,0,0,124, - 1,0,131,3,0,125,6,0,124,6,0,100,1,0,107,8, - 0,114,144,0,119,66,0,89,110,19,0,88,124,5,0,124, - 0,0,124,1,0,124,2,0,131,3,0,125,6,0,87,100, - 1,0,81,82,88,124,6,0,100,1,0,107,9,0,114,66, - 0,124,3,0,12,114,26,1,124,0,0,116,0,0,106,5, - 0,107,6,0,114,26,1,116,0,0,106,5,0,124,0,0, - 25,125,7,0,121,13,0,124,7,0,106,10,0,125,8,0, - 87,110,22,0,4,116,8,0,107,10,0,114,2,1,1,1, - 1,124,6,0,83,89,113,30,1,88,124,8,0,100,1,0, - 107,8,0,114,19,1,124,6,0,83,124,8,0,83,113,66, - 0,124,6,0,83,113,66,0,87,100,1,0,83,100,1,0, - 83,41,3,122,23,70,105,110,100,32,97,32,109,111,100,117, - 108,101,39,115,32,108,111,97,100,101,114,46,78,122,22,115, - 121,115,46,109,101,116,97,95,112,97,116,104,32,105,115,32, - 101,109,112,116,121,41,11,114,14,0,0,0,218,9,109,101, - 116,97,95,112,97,116,104,114,142,0,0,0,114,143,0,0, - 0,218,13,73,109,112,111,114,116,87,97,114,110,105,110,103, - 114,21,0,0,0,114,166,0,0,0,114,155,0,0,0,114, - 96,0,0,0,114,174,0,0,0,114,95,0,0,0,41,9, - 114,15,0,0,0,114,153,0,0,0,114,154,0,0,0,90, - 9,105,115,95,114,101,108,111,97,100,114,173,0,0,0,114, - 155,0,0,0,114,88,0,0,0,114,89,0,0,0,114,95, - 0,0,0,114,10,0,0,0,114,10,0,0,0,114,11,0, - 0,0,218,10,95,102,105,110,100,95,115,112,101,99,111,3, - 0,0,115,48,0,0,0,0,2,25,1,16,4,15,1,16, - 1,10,1,3,1,13,1,13,1,18,1,12,1,8,2,25, - 1,12,2,22,1,13,1,3,1,13,1,13,4,9,2,12, - 1,4,2,7,2,8,2,114,177,0,0,0,99,3,0,0, - 0,0,0,0,0,4,0,0,0,4,0,0,0,67,0,0, - 0,115,206,0,0,0,116,0,0,124,0,0,116,1,0,131, - 2,0,115,42,0,116,2,0,100,1,0,106,3,0,116,4, - 0,124,0,0,131,1,0,131,1,0,131,1,0,130,1,0, - 124,2,0,100,2,0,107,0,0,114,66,0,116,5,0,100, - 3,0,131,1,0,130,1,0,124,2,0,100,2,0,107,4, - 0,114,171,0,116,0,0,124,1,0,116,1,0,131,2,0, - 115,108,0,116,2,0,100,4,0,131,1,0,130,1,0,110, - 63,0,124,1,0,115,129,0,116,6,0,100,5,0,131,1, - 0,130,1,0,110,42,0,124,1,0,116,7,0,106,8,0, - 107,7,0,114,171,0,100,6,0,125,3,0,116,9,0,124, - 3,0,106,3,0,124,1,0,131,1,0,131,1,0,130,1, - 0,124,0,0,12,114,202,0,124,2,0,100,2,0,107,2, - 0,114,202,0,116,5,0,100,7,0,131,1,0,130,1,0, - 100,8,0,83,41,9,122,28,86,101,114,105,102,121,32,97, - 114,103,117,109,101,110,116,115,32,97,114,101,32,34,115,97, - 110,101,34,46,122,31,109,111,100,117,108,101,32,110,97,109, - 101,32,109,117,115,116,32,98,101,32,115,116,114,44,32,110, - 111,116,32,123,125,114,33,0,0,0,122,18,108,101,118,101, - 108,32,109,117,115,116,32,98,101,32,62,61,32,48,122,31, - 95,95,112,97,99,107,97,103,101,95,95,32,110,111,116,32, - 115,101,116,32,116,111,32,97,32,115,116,114,105,110,103,122, - 54,97,116,116,101,109,112,116,101,100,32,114,101,108,97,116, - 105,118,101,32,105,109,112,111,114,116,32,119,105,116,104,32, - 110,111,32,107,110,111,119,110,32,112,97,114,101,110,116,32, - 112,97,99,107,97,103,101,122,61,80,97,114,101,110,116,32, - 109,111,100,117,108,101,32,123,33,114,125,32,110,111,116,32, - 108,111,97,100,101,100,44,32,99,97,110,110,111,116,32,112, - 101,114,102,111,114,109,32,114,101,108,97,116,105,118,101,32, - 105,109,112,111,114,116,122,17,69,109,112,116,121,32,109,111, - 100,117,108,101,32,110,97,109,101,78,41,10,218,10,105,115, - 105,110,115,116,97,110,99,101,218,3,115,116,114,218,9,84, - 121,112,101,69,114,114,111,114,114,50,0,0,0,114,13,0, - 0,0,114,169,0,0,0,114,77,0,0,0,114,14,0,0, - 0,114,21,0,0,0,218,11,83,121,115,116,101,109,69,114, - 114,111,114,41,4,114,15,0,0,0,114,170,0,0,0,114, - 171,0,0,0,114,148,0,0,0,114,10,0,0,0,114,10, - 0,0,0,114,11,0,0,0,218,13,95,115,97,110,105,116, - 121,95,99,104,101,99,107,151,3,0,0,115,28,0,0,0, - 0,2,15,1,27,1,12,1,12,1,12,1,15,1,15,1, - 6,1,15,2,15,1,6,2,21,1,19,1,114,182,0,0, - 0,122,16,78,111,32,109,111,100,117,108,101,32,110,97,109, - 101,100,32,122,4,123,33,114,125,99,2,0,0,0,0,0, - 0,0,8,0,0,0,12,0,0,0,67,0,0,0,115,40, - 1,0,0,100,0,0,125,2,0,124,0,0,106,0,0,100, - 1,0,131,1,0,100,2,0,25,125,3,0,124,3,0,114, - 175,0,124,3,0,116,1,0,106,2,0,107,7,0,114,59, - 0,116,3,0,124,1,0,124,3,0,131,2,0,1,124,0, - 0,116,1,0,106,2,0,107,6,0,114,85,0,116,1,0, - 106,2,0,124,0,0,25,83,116,1,0,106,2,0,124,3, - 0,25,125,4,0,121,13,0,124,4,0,106,4,0,125,2, - 0,87,110,61,0,4,116,5,0,107,10,0,114,174,0,1, - 1,1,116,6,0,100,3,0,23,106,7,0,124,0,0,124, - 3,0,131,2,0,125,5,0,116,8,0,124,5,0,100,4, - 0,124,0,0,131,1,1,100,0,0,130,2,0,89,110,1, - 0,88,116,9,0,124,0,0,124,2,0,131,2,0,125,6, - 0,124,6,0,100,0,0,107,8,0,114,232,0,116,8,0, - 116,6,0,106,7,0,124,0,0,131,1,0,100,4,0,124, - 0,0,131,1,1,130,1,0,110,12,0,116,10,0,124,6, - 0,131,1,0,125,7,0,124,3,0,114,36,1,116,1,0, - 106,2,0,124,3,0,25,125,4,0,116,11,0,124,4,0, - 124,0,0,106,0,0,100,1,0,131,1,0,100,5,0,25, - 124,7,0,131,3,0,1,124,7,0,83,41,6,78,114,121, - 0,0,0,114,33,0,0,0,122,23,59,32,123,33,114,125, - 32,105,115,32,110,111,116,32,97,32,112,97,99,107,97,103, - 101,114,15,0,0,0,114,141,0,0,0,41,12,114,122,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,65,0,0, - 0,114,131,0,0,0,114,96,0,0,0,218,8,95,69,82, - 82,95,77,83,71,114,50,0,0,0,114,77,0,0,0,114, - 177,0,0,0,114,150,0,0,0,114,5,0,0,0,41,8, - 114,15,0,0,0,218,7,105,109,112,111,114,116,95,114,153, - 0,0,0,114,123,0,0,0,90,13,112,97,114,101,110,116, - 95,109,111,100,117,108,101,114,148,0,0,0,114,88,0,0, - 0,114,89,0,0,0,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,23,95,102,105,110,100,95,97,110,100, - 95,108,111,97,100,95,117,110,108,111,99,107,101,100,174,3, - 0,0,115,42,0,0,0,0,1,6,1,19,1,6,1,15, - 1,13,2,15,1,11,1,13,1,3,1,13,1,13,1,22, - 1,26,1,15,1,12,1,30,2,12,1,6,2,13,1,29, - 1,114,185,0,0,0,99,2,0,0,0,0,0,0,0,2, - 0,0,0,10,0,0,0,67,0,0,0,115,37,0,0,0, - 116,0,0,124,0,0,131,1,0,143,18,0,1,116,1,0, - 124,0,0,124,1,0,131,2,0,83,87,100,1,0,81,82, - 88,100,1,0,83,41,2,122,54,70,105,110,100,32,97,110, - 100,32,108,111,97,100,32,116,104,101,32,109,111,100,117,108, - 101,44,32,97,110,100,32,114,101,108,101,97,115,101,32,116, - 104,101,32,105,109,112,111,114,116,32,108,111,99,107,46,78, - 41,2,114,54,0,0,0,114,185,0,0,0,41,2,114,15, - 0,0,0,114,184,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,11,0,0,0,218,14,95,102,105,110,100,95,97, - 110,100,95,108,111,97,100,201,3,0,0,115,4,0,0,0, - 0,2,13,1,114,186,0,0,0,114,33,0,0,0,99,3, - 0,0,0,0,0,0,0,5,0,0,0,4,0,0,0,67, - 0,0,0,115,166,0,0,0,116,0,0,124,0,0,124,1, - 0,124,2,0,131,3,0,1,124,2,0,100,1,0,107,4, - 0,114,46,0,116,1,0,124,0,0,124,1,0,124,2,0, - 131,3,0,125,0,0,116,2,0,106,3,0,131,0,0,1, - 124,0,0,116,4,0,106,5,0,107,7,0,114,84,0,116, - 6,0,124,0,0,116,7,0,131,2,0,83,116,4,0,106, - 5,0,124,0,0,25,125,3,0,124,3,0,100,2,0,107, - 8,0,114,152,0,116,2,0,106,8,0,131,0,0,1,100, - 3,0,106,9,0,124,0,0,131,1,0,125,4,0,116,10, - 0,124,4,0,100,4,0,124,0,0,131,1,1,130,1,0, - 116,11,0,124,0,0,131,1,0,1,124,3,0,83,41,5, - 97,50,1,0,0,73,109,112,111,114,116,32,97,110,100,32, - 114,101,116,117,114,110,32,116,104,101,32,109,111,100,117,108, - 101,32,98,97,115,101,100,32,111,110,32,105,116,115,32,110, - 97,109,101,44,32,116,104,101,32,112,97,99,107,97,103,101, - 32,116,104,101,32,99,97,108,108,32,105,115,10,32,32,32, - 32,98,101,105,110,103,32,109,97,100,101,32,102,114,111,109, - 44,32,97,110,100,32,116,104,101,32,108,101,118,101,108,32, - 97,100,106,117,115,116,109,101,110,116,46,10,10,32,32,32, - 32,84,104,105,115,32,102,117,110,99,116,105,111,110,32,114, - 101,112,114,101,115,101,110,116,115,32,116,104,101,32,103,114, - 101,97,116,101,115,116,32,99,111,109,109,111,110,32,100,101, - 110,111,109,105,110,97,116,111,114,32,111,102,32,102,117,110, - 99,116,105,111,110,97,108,105,116,121,10,32,32,32,32,98, - 101,116,119,101,101,110,32,105,109,112,111,114,116,95,109,111, - 100,117,108,101,32,97,110,100,32,95,95,105,109,112,111,114, - 116,95,95,46,32,84,104,105,115,32,105,110,99,108,117,100, - 101,115,32,115,101,116,116,105,110,103,32,95,95,112,97,99, - 107,97,103,101,95,95,32,105,102,10,32,32,32,32,116,104, - 101,32,108,111,97,100,101,114,32,100,105,100,32,110,111,116, - 46,10,10,32,32,32,32,114,33,0,0,0,78,122,40,105, - 109,112,111,114,116,32,111,102,32,123,125,32,104,97,108,116, - 101,100,59,32,78,111,110,101,32,105,110,32,115,121,115,46, - 109,111,100,117,108,101,115,114,15,0,0,0,41,12,114,182, - 0,0,0,114,172,0,0,0,114,57,0,0,0,114,146,0, - 0,0,114,14,0,0,0,114,21,0,0,0,114,186,0,0, - 0,218,11,95,103,99,100,95,105,109,112,111,114,116,114,58, - 0,0,0,114,50,0,0,0,114,77,0,0,0,114,63,0, - 0,0,41,5,114,15,0,0,0,114,170,0,0,0,114,171, - 0,0,0,114,89,0,0,0,114,74,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,114,187,0,0, - 0,207,3,0,0,115,28,0,0,0,0,9,16,1,12,1, - 18,1,10,1,15,1,13,1,13,1,12,1,10,1,6,1, - 9,1,18,1,10,1,114,187,0,0,0,99,3,0,0,0, - 0,0,0,0,6,0,0,0,17,0,0,0,67,0,0,0, - 115,239,0,0,0,116,0,0,124,0,0,100,1,0,131,2, - 0,114,235,0,100,2,0,124,1,0,107,6,0,114,83,0, - 116,1,0,124,1,0,131,1,0,125,1,0,124,1,0,106, - 2,0,100,2,0,131,1,0,1,116,0,0,124,0,0,100, - 3,0,131,2,0,114,83,0,124,1,0,106,3,0,124,0, - 0,106,4,0,131,1,0,1,120,149,0,124,1,0,68,93, - 141,0,125,3,0,116,0,0,124,0,0,124,3,0,131,2, - 0,115,90,0,100,4,0,106,5,0,124,0,0,106,6,0, - 124,3,0,131,2,0,125,4,0,121,17,0,116,7,0,124, - 2,0,124,4,0,131,2,0,1,87,113,90,0,4,116,8, - 0,107,10,0,114,230,0,1,125,5,0,1,122,47,0,116, - 9,0,124,5,0,131,1,0,106,10,0,116,11,0,131,1, - 0,114,209,0,124,5,0,106,12,0,124,4,0,107,2,0, - 114,209,0,119,90,0,130,0,0,87,89,100,5,0,100,5, - 0,125,5,0,126,5,0,88,113,90,0,88,113,90,0,87, - 124,0,0,83,41,6,122,238,70,105,103,117,114,101,32,111, - 117,116,32,119,104,97,116,32,95,95,105,109,112,111,114,116, - 95,95,32,115,104,111,117,108,100,32,114,101,116,117,114,110, - 46,10,10,32,32,32,32,84,104,101,32,105,109,112,111,114, - 116,95,32,112,97,114,97,109,101,116,101,114,32,105,115,32, - 97,32,99,97,108,108,97,98,108,101,32,119,104,105,99,104, - 32,116,97,107,101,115,32,116,104,101,32,110,97,109,101,32, - 111,102,32,109,111,100,117,108,101,32,116,111,10,32,32,32, - 32,105,109,112,111,114,116,46,32,73,116,32,105,115,32,114, - 101,113,117,105,114,101,100,32,116,111,32,100,101,99,111,117, - 112,108,101,32,116,104,101,32,102,117,110,99,116,105,111,110, - 32,102,114,111,109,32,97,115,115,117,109,105,110,103,32,105, - 109,112,111,114,116,108,105,98,39,115,10,32,32,32,32,105, - 109,112,111,114,116,32,105,109,112,108,101,109,101,110,116,97, - 116,105,111,110,32,105,115,32,100,101,115,105,114,101,100,46, - 10,10,32,32,32,32,114,131,0,0,0,250,1,42,218,7, - 95,95,97,108,108,95,95,122,5,123,125,46,123,125,78,41, - 13,114,4,0,0,0,114,130,0,0,0,218,6,114,101,109, - 111,118,101,218,6,101,120,116,101,110,100,114,189,0,0,0, - 114,50,0,0,0,114,1,0,0,0,114,65,0,0,0,114, - 77,0,0,0,114,179,0,0,0,114,71,0,0,0,218,15, - 95,69,82,82,95,77,83,71,95,80,82,69,70,73,88,114, - 15,0,0,0,41,6,114,89,0,0,0,218,8,102,114,111, - 109,108,105,115,116,114,184,0,0,0,218,1,120,90,9,102, - 114,111,109,95,110,97,109,101,90,3,101,120,99,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,16,95,104, - 97,110,100,108,101,95,102,114,111,109,108,105,115,116,231,3, - 0,0,115,34,0,0,0,0,10,15,1,12,1,12,1,13, - 1,15,1,16,1,13,1,15,1,21,1,3,1,17,1,18, - 4,21,1,15,1,3,1,26,1,114,195,0,0,0,99,1, - 0,0,0,0,0,0,0,3,0,0,0,7,0,0,0,67, - 0,0,0,115,214,0,0,0,124,0,0,106,0,0,100,1, - 0,131,1,0,125,1,0,124,0,0,106,0,0,100,2,0, - 131,1,0,125,2,0,124,1,0,100,3,0,107,9,0,114, - 128,0,124,2,0,100,3,0,107,9,0,114,124,0,124,1, - 0,124,2,0,106,1,0,107,3,0,114,124,0,116,2,0, - 106,3,0,100,4,0,106,4,0,100,5,0,124,1,0,155, - 2,0,100,6,0,124,2,0,106,1,0,155,2,0,100,7, - 0,103,5,0,131,1,0,116,5,0,100,8,0,100,9,0, - 131,2,1,1,124,1,0,83,124,2,0,100,3,0,107,9, - 0,114,147,0,124,2,0,106,1,0,83,116,2,0,106,3, - 0,100,10,0,116,5,0,100,8,0,100,9,0,131,2,1, - 1,124,0,0,100,11,0,25,125,1,0,100,12,0,124,0, - 0,107,7,0,114,210,0,124,1,0,106,6,0,100,13,0, - 131,1,0,100,14,0,25,125,1,0,124,1,0,83,41,15, - 122,167,67,97,108,99,117,108,97,116,101,32,119,104,97,116, - 32,95,95,112,97,99,107,97,103,101,95,95,32,115,104,111, - 117,108,100,32,98,101,46,10,10,32,32,32,32,95,95,112, - 97,99,107,97,103,101,95,95,32,105,115,32,110,111,116,32, - 103,117,97,114,97,110,116,101,101,100,32,116,111,32,98,101, - 32,100,101,102,105,110,101,100,32,111,114,32,99,111,117,108, - 100,32,98,101,32,115,101,116,32,116,111,32,78,111,110,101, - 10,32,32,32,32,116,111,32,114,101,112,114,101,115,101,110, - 116,32,116,104,97,116,32,105,116,115,32,112,114,111,112,101, - 114,32,118,97,108,117,101,32,105,115,32,117,110,107,110,111, - 119,110,46,10,10,32,32,32,32,114,134,0,0,0,114,95, - 0,0,0,78,218,0,122,32,95,95,112,97,99,107,97,103, - 101,95,95,32,33,61,32,95,95,115,112,101,99,95,95,46, - 112,97,114,101,110,116,32,40,122,4,32,33,61,32,250,1, - 41,114,140,0,0,0,233,3,0,0,0,122,89,99,97,110, - 39,116,32,114,101,115,111,108,118,101,32,112,97,99,107,97, - 103,101,32,102,114,111,109,32,95,95,115,112,101,99,95,95, - 32,111,114,32,95,95,112,97,99,107,97,103,101,95,95,44, - 32,102,97,108,108,105,110,103,32,98,97,99,107,32,111,110, - 32,95,95,110,97,109,101,95,95,32,97,110,100,32,95,95, - 112,97,116,104,95,95,114,1,0,0,0,114,131,0,0,0, - 114,121,0,0,0,114,33,0,0,0,41,7,114,42,0,0, - 0,114,123,0,0,0,114,142,0,0,0,114,143,0,0,0, - 114,115,0,0,0,114,176,0,0,0,114,122,0,0,0,41, - 3,218,7,103,108,111,98,97,108,115,114,170,0,0,0,114, - 88,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,17,95,99,97,108,99,95,95,95,112,97,99, - 107,97,103,101,95,95,7,4,0,0,115,30,0,0,0,0, - 7,15,1,15,1,12,1,27,1,42,2,13,1,4,1,12, - 1,7,2,9,2,13,1,10,1,12,1,19,1,114,200,0, - 0,0,99,5,0,0,0,0,0,0,0,9,0,0,0,5, - 0,0,0,67,0,0,0,115,227,0,0,0,124,4,0,100, - 1,0,107,2,0,114,27,0,116,0,0,124,0,0,131,1, - 0,125,5,0,110,54,0,124,1,0,100,2,0,107,9,0, - 114,45,0,124,1,0,110,3,0,105,0,0,125,6,0,116, - 1,0,124,6,0,131,1,0,125,7,0,116,0,0,124,0, - 0,124,7,0,124,4,0,131,3,0,125,5,0,124,3,0, - 115,207,0,124,4,0,100,1,0,107,2,0,114,122,0,116, - 0,0,124,0,0,106,2,0,100,3,0,131,1,0,100,1, - 0,25,131,1,0,83,124,0,0,115,132,0,124,5,0,83, - 116,3,0,124,0,0,131,1,0,116,3,0,124,0,0,106, - 2,0,100,3,0,131,1,0,100,1,0,25,131,1,0,24, - 125,8,0,116,4,0,106,5,0,124,5,0,106,6,0,100, - 2,0,116,3,0,124,5,0,106,6,0,131,1,0,124,8, - 0,24,133,2,0,25,25,83,110,16,0,116,7,0,124,5, - 0,124,3,0,116,0,0,131,3,0,83,100,2,0,83,41, - 4,97,214,1,0,0,73,109,112,111,114,116,32,97,32,109, - 111,100,117,108,101,46,10,10,32,32,32,32,84,104,101,32, - 39,103,108,111,98,97,108,115,39,32,97,114,103,117,109,101, - 110,116,32,105,115,32,117,115,101,100,32,116,111,32,105,110, - 102,101,114,32,119,104,101,114,101,32,116,104,101,32,105,109, - 112,111,114,116,32,105,115,32,111,99,99,117,114,105,110,103, - 32,102,114,111,109,10,32,32,32,32,116,111,32,104,97,110, - 100,108,101,32,114,101,108,97,116,105,118,101,32,105,109,112, - 111,114,116,115,46,32,84,104,101,32,39,108,111,99,97,108, - 115,39,32,97,114,103,117,109,101,110,116,32,105,115,32,105, - 103,110,111,114,101,100,46,32,84,104,101,10,32,32,32,32, - 39,102,114,111,109,108,105,115,116,39,32,97,114,103,117,109, - 101,110,116,32,115,112,101,99,105,102,105,101,115,32,119,104, - 97,116,32,115,104,111,117,108,100,32,101,120,105,115,116,32, - 97,115,32,97,116,116,114,105,98,117,116,101,115,32,111,110, - 32,116,104,101,32,109,111,100,117,108,101,10,32,32,32,32, - 98,101,105,110,103,32,105,109,112,111,114,116,101,100,32,40, - 101,46,103,46,32,96,96,102,114,111,109,32,109,111,100,117, - 108,101,32,105,109,112,111,114,116,32,60,102,114,111,109,108, - 105,115,116,62,96,96,41,46,32,32,84,104,101,32,39,108, - 101,118,101,108,39,10,32,32,32,32,97,114,103,117,109,101, - 110,116,32,114,101,112,114,101,115,101,110,116,115,32,116,104, - 101,32,112,97,99,107,97,103,101,32,108,111,99,97,116,105, - 111,110,32,116,111,32,105,109,112,111,114,116,32,102,114,111, - 109,32,105,110,32,97,32,114,101,108,97,116,105,118,101,10, - 32,32,32,32,105,109,112,111,114,116,32,40,101,46,103,46, - 32,96,96,102,114,111,109,32,46,46,112,107,103,32,105,109, - 112,111,114,116,32,109,111,100,96,96,32,119,111,117,108,100, - 32,104,97,118,101,32,97,32,39,108,101,118,101,108,39,32, - 111,102,32,50,41,46,10,10,32,32,32,32,114,33,0,0, - 0,78,114,121,0,0,0,41,8,114,187,0,0,0,114,200, - 0,0,0,218,9,112,97,114,116,105,116,105,111,110,114,168, - 0,0,0,114,14,0,0,0,114,21,0,0,0,114,1,0, - 0,0,114,195,0,0,0,41,9,114,15,0,0,0,114,199, - 0,0,0,218,6,108,111,99,97,108,115,114,193,0,0,0, - 114,171,0,0,0,114,89,0,0,0,90,8,103,108,111,98, - 97,108,115,95,114,170,0,0,0,90,7,99,117,116,95,111, - 102,102,114,10,0,0,0,114,10,0,0,0,114,11,0,0, - 0,218,10,95,95,105,109,112,111,114,116,95,95,34,4,0, - 0,115,26,0,0,0,0,11,12,1,15,2,24,1,12,1, - 18,1,6,3,12,1,23,1,6,1,4,4,35,3,40,2, - 114,203,0,0,0,99,1,0,0,0,0,0,0,0,2,0, - 0,0,3,0,0,0,67,0,0,0,115,53,0,0,0,116, - 0,0,106,1,0,124,0,0,131,1,0,125,1,0,124,1, - 0,100,0,0,107,8,0,114,43,0,116,2,0,100,1,0, - 124,0,0,23,131,1,0,130,1,0,116,3,0,124,1,0, - 131,1,0,83,41,2,78,122,25,110,111,32,98,117,105,108, - 116,45,105,110,32,109,111,100,117,108,101,32,110,97,109,101, - 100,32,41,4,114,151,0,0,0,114,155,0,0,0,114,77, - 0,0,0,114,150,0,0,0,41,2,114,15,0,0,0,114, - 88,0,0,0,114,10,0,0,0,114,10,0,0,0,114,11, - 0,0,0,218,18,95,98,117,105,108,116,105,110,95,102,114, - 111,109,95,110,97,109,101,69,4,0,0,115,8,0,0,0, - 0,1,15,1,12,1,16,1,114,204,0,0,0,99,2,0, - 0,0,0,0,0,0,12,0,0,0,12,0,0,0,67,0, - 0,0,115,74,1,0,0,124,1,0,97,0,0,124,0,0, - 97,1,0,116,2,0,116,1,0,131,1,0,125,2,0,120, - 123,0,116,1,0,106,3,0,106,4,0,131,0,0,68,93, - 106,0,92,2,0,125,3,0,125,4,0,116,5,0,124,4, - 0,124,2,0,131,2,0,114,40,0,124,3,0,116,1,0, - 106,6,0,107,6,0,114,91,0,116,7,0,125,5,0,110, - 27,0,116,0,0,106,8,0,124,3,0,131,1,0,114,40, - 0,116,9,0,125,5,0,110,3,0,113,40,0,116,10,0, - 124,4,0,124,5,0,131,2,0,125,6,0,116,11,0,124, - 6,0,124,4,0,131,2,0,1,113,40,0,87,116,1,0, - 106,3,0,116,12,0,25,125,7,0,120,73,0,100,5,0, - 68,93,65,0,125,8,0,124,8,0,116,1,0,106,3,0, - 107,7,0,114,206,0,116,13,0,124,8,0,131,1,0,125, - 9,0,110,13,0,116,1,0,106,3,0,124,8,0,25,125, - 9,0,116,14,0,124,7,0,124,8,0,124,9,0,131,3, - 0,1,113,170,0,87,121,16,0,116,13,0,100,2,0,131, - 1,0,125,10,0,87,110,24,0,4,116,15,0,107,10,0, - 114,25,1,1,1,1,100,3,0,125,10,0,89,110,1,0, - 88,116,14,0,124,7,0,100,2,0,124,10,0,131,3,0, - 1,116,13,0,100,4,0,131,1,0,125,11,0,116,14,0, - 124,7,0,100,4,0,124,11,0,131,3,0,1,100,3,0, - 83,41,6,122,250,83,101,116,117,112,32,105,109,112,111,114, - 116,108,105,98,32,98,121,32,105,109,112,111,114,116,105,110, - 103,32,110,101,101,100,101,100,32,98,117,105,108,116,45,105, - 110,32,109,111,100,117,108,101,115,32,97,110,100,32,105,110, - 106,101,99,116,105,110,103,32,116,104,101,109,10,32,32,32, - 32,105,110,116,111,32,116,104,101,32,103,108,111,98,97,108, - 32,110,97,109,101,115,112,97,99,101,46,10,10,32,32,32, - 32,65,115,32,115,121,115,32,105,115,32,110,101,101,100,101, - 100,32,102,111,114,32,115,121,115,46,109,111,100,117,108,101, - 115,32,97,99,99,101,115,115,32,97,110,100,32,95,105,109, - 112,32,105,115,32,110,101,101,100,101,100,32,116,111,32,108, - 111,97,100,32,98,117,105,108,116,45,105,110,10,32,32,32, - 32,109,111,100,117,108,101,115,44,32,116,104,111,115,101,32, - 116,119,111,32,109,111,100,117,108,101,115,32,109,117,115,116, - 32,98,101,32,101,120,112,108,105,99,105,116,108,121,32,112, - 97,115,115,101,100,32,105,110,46,10,10,32,32,32,32,114, - 142,0,0,0,114,34,0,0,0,78,114,62,0,0,0,41, - 1,122,9,95,119,97,114,110,105,110,103,115,41,16,114,57, - 0,0,0,114,14,0,0,0,114,13,0,0,0,114,21,0, - 0,0,218,5,105,116,101,109,115,114,178,0,0,0,114,76, - 0,0,0,114,151,0,0,0,114,82,0,0,0,114,161,0, - 0,0,114,132,0,0,0,114,137,0,0,0,114,1,0,0, - 0,114,204,0,0,0,114,5,0,0,0,114,77,0,0,0, - 41,12,218,10,115,121,115,95,109,111,100,117,108,101,218,11, - 95,105,109,112,95,109,111,100,117,108,101,90,11,109,111,100, - 117,108,101,95,116,121,112,101,114,15,0,0,0,114,89,0, - 0,0,114,99,0,0,0,114,88,0,0,0,90,11,115,101, - 108,102,95,109,111,100,117,108,101,90,12,98,117,105,108,116, - 105,110,95,110,97,109,101,90,14,98,117,105,108,116,105,110, - 95,109,111,100,117,108,101,90,13,116,104,114,101,97,100,95, - 109,111,100,117,108,101,90,14,119,101,97,107,114,101,102,95, - 109,111,100,117,108,101,114,10,0,0,0,114,10,0,0,0, - 114,11,0,0,0,218,6,95,115,101,116,117,112,76,4,0, - 0,115,50,0,0,0,0,9,6,1,6,3,12,1,28,1, - 15,1,15,1,9,1,15,1,9,2,3,1,15,1,17,3, - 13,1,13,1,15,1,15,2,13,1,20,3,3,1,16,1, - 13,2,11,1,16,3,12,1,114,208,0,0,0,99,2,0, - 0,0,0,0,0,0,3,0,0,0,3,0,0,0,67,0, - 0,0,115,87,0,0,0,116,0,0,124,0,0,124,1,0, - 131,2,0,1,116,1,0,106,2,0,106,3,0,116,4,0, - 131,1,0,1,116,1,0,106,2,0,106,3,0,116,5,0, - 131,1,0,1,100,1,0,100,2,0,108,6,0,125,2,0, - 124,2,0,97,7,0,124,2,0,106,8,0,116,1,0,106, - 9,0,116,10,0,25,131,1,0,1,100,2,0,83,41,3, - 122,50,73,110,115,116,97,108,108,32,105,109,112,111,114,116, - 108,105,98,32,97,115,32,116,104,101,32,105,109,112,108,101, - 109,101,110,116,97,116,105,111,110,32,111,102,32,105,109,112, - 111,114,116,46,114,33,0,0,0,78,41,11,114,208,0,0, - 0,114,14,0,0,0,114,175,0,0,0,114,113,0,0,0, - 114,151,0,0,0,114,161,0,0,0,218,26,95,102,114,111, - 122,101,110,95,105,109,112,111,114,116,108,105,98,95,101,120, - 116,101,114,110,97,108,114,119,0,0,0,218,8,95,105,110, - 115,116,97,108,108,114,21,0,0,0,114,1,0,0,0,41, - 3,114,206,0,0,0,114,207,0,0,0,114,209,0,0,0, - 114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,114, - 210,0,0,0,123,4,0,0,115,12,0,0,0,0,2,13, - 2,16,1,16,3,12,1,6,1,114,210,0,0,0,41,51, - 114,3,0,0,0,114,119,0,0,0,114,12,0,0,0,114, - 16,0,0,0,114,17,0,0,0,114,59,0,0,0,114,41, - 0,0,0,114,48,0,0,0,114,31,0,0,0,114,32,0, - 0,0,114,53,0,0,0,114,54,0,0,0,114,56,0,0, - 0,114,63,0,0,0,114,65,0,0,0,114,75,0,0,0, - 114,81,0,0,0,114,84,0,0,0,114,90,0,0,0,114, - 101,0,0,0,114,102,0,0,0,114,106,0,0,0,114,85, - 0,0,0,218,6,111,98,106,101,99,116,90,9,95,80,79, - 80,85,76,65,84,69,114,132,0,0,0,114,137,0,0,0, - 114,145,0,0,0,114,97,0,0,0,114,86,0,0,0,114, - 149,0,0,0,114,150,0,0,0,114,87,0,0,0,114,151, - 0,0,0,114,161,0,0,0,114,166,0,0,0,114,172,0, - 0,0,114,174,0,0,0,114,177,0,0,0,114,182,0,0, - 0,114,192,0,0,0,114,183,0,0,0,114,185,0,0,0, - 114,186,0,0,0,114,187,0,0,0,114,195,0,0,0,114, - 200,0,0,0,114,203,0,0,0,114,204,0,0,0,114,208, - 0,0,0,114,210,0,0,0,114,10,0,0,0,114,10,0, - 0,0,114,10,0,0,0,114,11,0,0,0,218,8,60,109, - 111,100,117,108,101,62,8,0,0,0,115,96,0,0,0,6, - 17,6,2,12,8,12,4,19,20,6,2,6,3,22,4,19, - 68,19,21,19,19,12,19,12,19,12,11,18,8,12,11,12, - 12,12,16,12,36,19,27,19,101,24,26,9,3,18,45,18, - 60,12,18,12,17,12,25,12,29,12,23,12,16,19,73,19, - 77,19,13,12,9,12,9,15,40,12,20,6,1,10,2,12, - 27,12,6,18,24,12,32,12,27,24,35,12,7,12,47, + 0,0,0,0,0,0,10,0,0,0,27,0,0,0,67,0, + 0,0,115,53,1,0,0,116,0,0,106,1,0,125,3,0, + 124,3,0,100,1,0,107,8,0,114,33,0,116,2,0,100, + 2,0,131,1,0,130,1,0,124,3,0,115,55,0,116,3, + 0,106,4,0,100,3,0,116,5,0,131,2,0,1,124,0, + 0,116,0,0,106,6,0,107,6,0,125,4,0,120,232,0, + 124,3,0,68,93,220,0,125,5,0,116,7,0,131,0,0, + 143,90,0,1,121,13,0,124,5,0,106,8,0,125,6,0, + 87,110,51,0,4,116,9,0,107,10,0,114,159,0,1,1, + 1,116,10,0,124,5,0,124,0,0,124,1,0,131,3,0, + 125,7,0,124,7,0,100,1,0,107,8,0,114,155,0,119, + 77,0,89,110,19,0,88,124,6,0,124,0,0,124,1,0, + 124,2,0,131,3,0,125,7,0,87,100,1,0,81,82,88, + 124,7,0,100,1,0,107,9,0,114,77,0,124,4,0,12, + 114,37,1,124,0,0,116,0,0,106,6,0,107,6,0,114, + 37,1,116,0,0,106,6,0,124,0,0,25,125,8,0,121, + 13,0,124,8,0,106,11,0,125,9,0,87,110,22,0,4, + 116,9,0,107,10,0,114,13,1,1,1,1,124,7,0,83, + 89,113,41,1,88,124,9,0,100,1,0,107,8,0,114,30, + 1,124,7,0,83,124,9,0,83,113,77,0,124,7,0,83, + 113,77,0,87,100,1,0,83,100,1,0,83,41,4,122,23, + 70,105,110,100,32,97,32,109,111,100,117,108,101,39,115,32, + 108,111,97,100,101,114,46,78,122,53,115,121,115,46,109,101, + 116,97,95,112,97,116,104,32,105,115,32,78,111,110,101,44, + 32,80,121,116,104,111,110,32,105,115,32,108,105,107,101,108, + 121,32,115,104,117,116,116,105,110,103,32,100,111,119,110,122, + 22,115,121,115,46,109,101,116,97,95,112,97,116,104,32,105, + 115,32,101,109,112,116,121,41,12,114,14,0,0,0,218,9, + 109,101,116,97,95,112,97,116,104,114,77,0,0,0,114,142, + 0,0,0,114,143,0,0,0,218,13,73,109,112,111,114,116, + 87,97,114,110,105,110,103,114,21,0,0,0,114,166,0,0, + 0,114,155,0,0,0,114,96,0,0,0,114,174,0,0,0, + 114,95,0,0,0,41,10,114,15,0,0,0,114,153,0,0, + 0,114,154,0,0,0,114,175,0,0,0,90,9,105,115,95, + 114,101,108,111,97,100,114,173,0,0,0,114,155,0,0,0, + 114,88,0,0,0,114,89,0,0,0,114,95,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,10, + 95,102,105,110,100,95,115,112,101,99,111,3,0,0,115,54, + 0,0,0,0,2,9,1,12,2,12,3,6,1,16,5,15, + 1,13,1,10,1,3,1,13,1,13,1,18,1,12,1,8, + 2,25,1,12,2,22,1,13,1,3,1,13,1,13,4,9, + 2,12,1,4,2,7,2,8,2,114,177,0,0,0,99,3, + 0,0,0,0,0,0,0,4,0,0,0,4,0,0,0,67, + 0,0,0,115,206,0,0,0,116,0,0,124,0,0,116,1, + 0,131,2,0,115,42,0,116,2,0,100,1,0,106,3,0, + 116,4,0,124,0,0,131,1,0,131,1,0,131,1,0,130, + 1,0,124,2,0,100,2,0,107,0,0,114,66,0,116,5, + 0,100,3,0,131,1,0,130,1,0,124,2,0,100,2,0, + 107,4,0,114,171,0,116,0,0,124,1,0,116,1,0,131, + 2,0,115,108,0,116,2,0,100,4,0,131,1,0,130,1, + 0,110,63,0,124,1,0,115,129,0,116,6,0,100,5,0, + 131,1,0,130,1,0,110,42,0,124,1,0,116,7,0,106, + 8,0,107,7,0,114,171,0,100,6,0,125,3,0,116,9, + 0,124,3,0,106,3,0,124,1,0,131,1,0,131,1,0, + 130,1,0,124,0,0,12,114,202,0,124,2,0,100,2,0, + 107,2,0,114,202,0,116,5,0,100,7,0,131,1,0,130, + 1,0,100,8,0,83,41,9,122,28,86,101,114,105,102,121, + 32,97,114,103,117,109,101,110,116,115,32,97,114,101,32,34, + 115,97,110,101,34,46,122,31,109,111,100,117,108,101,32,110, + 97,109,101,32,109,117,115,116,32,98,101,32,115,116,114,44, + 32,110,111,116,32,123,125,114,33,0,0,0,122,18,108,101, + 118,101,108,32,109,117,115,116,32,98,101,32,62,61,32,48, + 122,31,95,95,112,97,99,107,97,103,101,95,95,32,110,111, + 116,32,115,101,116,32,116,111,32,97,32,115,116,114,105,110, + 103,122,54,97,116,116,101,109,112,116,101,100,32,114,101,108, + 97,116,105,118,101,32,105,109,112,111,114,116,32,119,105,116, + 104,32,110,111,32,107,110,111,119,110,32,112,97,114,101,110, + 116,32,112,97,99,107,97,103,101,122,61,80,97,114,101,110, + 116,32,109,111,100,117,108,101,32,123,33,114,125,32,110,111, + 116,32,108,111,97,100,101,100,44,32,99,97,110,110,111,116, + 32,112,101,114,102,111,114,109,32,114,101,108,97,116,105,118, + 101,32,105,109,112,111,114,116,122,17,69,109,112,116,121,32, + 109,111,100,117,108,101,32,110,97,109,101,78,41,10,218,10, + 105,115,105,110,115,116,97,110,99,101,218,3,115,116,114,218, + 9,84,121,112,101,69,114,114,111,114,114,50,0,0,0,114, + 13,0,0,0,114,169,0,0,0,114,77,0,0,0,114,14, + 0,0,0,114,21,0,0,0,218,11,83,121,115,116,101,109, + 69,114,114,111,114,41,4,114,15,0,0,0,114,170,0,0, + 0,114,171,0,0,0,114,148,0,0,0,114,10,0,0,0, + 114,10,0,0,0,114,11,0,0,0,218,13,95,115,97,110, + 105,116,121,95,99,104,101,99,107,158,3,0,0,115,28,0, + 0,0,0,2,15,1,27,1,12,1,12,1,12,1,15,1, + 15,1,6,1,15,2,15,1,6,2,21,1,19,1,114,182, + 0,0,0,122,16,78,111,32,109,111,100,117,108,101,32,110, + 97,109,101,100,32,122,4,123,33,114,125,99,2,0,0,0, + 0,0,0,0,8,0,0,0,12,0,0,0,67,0,0,0, + 115,40,1,0,0,100,0,0,125,2,0,124,0,0,106,0, + 0,100,1,0,131,1,0,100,2,0,25,125,3,0,124,3, + 0,114,175,0,124,3,0,116,1,0,106,2,0,107,7,0, + 114,59,0,116,3,0,124,1,0,124,3,0,131,2,0,1, + 124,0,0,116,1,0,106,2,0,107,6,0,114,85,0,116, + 1,0,106,2,0,124,0,0,25,83,116,1,0,106,2,0, + 124,3,0,25,125,4,0,121,13,0,124,4,0,106,4,0, + 125,2,0,87,110,61,0,4,116,5,0,107,10,0,114,174, + 0,1,1,1,116,6,0,100,3,0,23,106,7,0,124,0, + 0,124,3,0,131,2,0,125,5,0,116,8,0,124,5,0, + 100,4,0,124,0,0,131,1,1,100,0,0,130,2,0,89, + 110,1,0,88,116,9,0,124,0,0,124,2,0,131,2,0, + 125,6,0,124,6,0,100,0,0,107,8,0,114,232,0,116, + 8,0,116,6,0,106,7,0,124,0,0,131,1,0,100,4, + 0,124,0,0,131,1,1,130,1,0,110,12,0,116,10,0, + 124,6,0,131,1,0,125,7,0,124,3,0,114,36,1,116, + 1,0,106,2,0,124,3,0,25,125,4,0,116,11,0,124, + 4,0,124,0,0,106,0,0,100,1,0,131,1,0,100,5, + 0,25,124,7,0,131,3,0,1,124,7,0,83,41,6,78, + 114,121,0,0,0,114,33,0,0,0,122,23,59,32,123,33, + 114,125,32,105,115,32,110,111,116,32,97,32,112,97,99,107, + 97,103,101,114,15,0,0,0,114,141,0,0,0,41,12,114, + 122,0,0,0,114,14,0,0,0,114,21,0,0,0,114,65, + 0,0,0,114,131,0,0,0,114,96,0,0,0,218,8,95, + 69,82,82,95,77,83,71,114,50,0,0,0,114,77,0,0, + 0,114,177,0,0,0,114,150,0,0,0,114,5,0,0,0, + 41,8,114,15,0,0,0,218,7,105,109,112,111,114,116,95, + 114,153,0,0,0,114,123,0,0,0,90,13,112,97,114,101, + 110,116,95,109,111,100,117,108,101,114,148,0,0,0,114,88, + 0,0,0,114,89,0,0,0,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,23,95,102,105,110,100,95,97, + 110,100,95,108,111,97,100,95,117,110,108,111,99,107,101,100, + 181,3,0,0,115,42,0,0,0,0,1,6,1,19,1,6, + 1,15,1,13,2,15,1,11,1,13,1,3,1,13,1,13, + 1,22,1,26,1,15,1,12,1,30,2,12,1,6,2,13, + 1,29,1,114,185,0,0,0,99,2,0,0,0,0,0,0, + 0,2,0,0,0,10,0,0,0,67,0,0,0,115,37,0, + 0,0,116,0,0,124,0,0,131,1,0,143,18,0,1,116, + 1,0,124,0,0,124,1,0,131,2,0,83,87,100,1,0, + 81,82,88,100,1,0,83,41,2,122,54,70,105,110,100,32, + 97,110,100,32,108,111,97,100,32,116,104,101,32,109,111,100, + 117,108,101,44,32,97,110,100,32,114,101,108,101,97,115,101, + 32,116,104,101,32,105,109,112,111,114,116,32,108,111,99,107, + 46,78,41,2,114,54,0,0,0,114,185,0,0,0,41,2, + 114,15,0,0,0,114,184,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,11,0,0,0,218,14,95,102,105,110,100, + 95,97,110,100,95,108,111,97,100,208,3,0,0,115,4,0, + 0,0,0,2,13,1,114,186,0,0,0,114,33,0,0,0, + 99,3,0,0,0,0,0,0,0,5,0,0,0,4,0,0, + 0,67,0,0,0,115,166,0,0,0,116,0,0,124,0,0, + 124,1,0,124,2,0,131,3,0,1,124,2,0,100,1,0, + 107,4,0,114,46,0,116,1,0,124,0,0,124,1,0,124, + 2,0,131,3,0,125,0,0,116,2,0,106,3,0,131,0, + 0,1,124,0,0,116,4,0,106,5,0,107,7,0,114,84, + 0,116,6,0,124,0,0,116,7,0,131,2,0,83,116,4, + 0,106,5,0,124,0,0,25,125,3,0,124,3,0,100,2, + 0,107,8,0,114,152,0,116,2,0,106,8,0,131,0,0, + 1,100,3,0,106,9,0,124,0,0,131,1,0,125,4,0, + 116,10,0,124,4,0,100,4,0,124,0,0,131,1,1,130, + 1,0,116,11,0,124,0,0,131,1,0,1,124,3,0,83, + 41,5,97,50,1,0,0,73,109,112,111,114,116,32,97,110, + 100,32,114,101,116,117,114,110,32,116,104,101,32,109,111,100, + 117,108,101,32,98,97,115,101,100,32,111,110,32,105,116,115, + 32,110,97,109,101,44,32,116,104,101,32,112,97,99,107,97, + 103,101,32,116,104,101,32,99,97,108,108,32,105,115,10,32, + 32,32,32,98,101,105,110,103,32,109,97,100,101,32,102,114, + 111,109,44,32,97,110,100,32,116,104,101,32,108,101,118,101, + 108,32,97,100,106,117,115,116,109,101,110,116,46,10,10,32, + 32,32,32,84,104,105,115,32,102,117,110,99,116,105,111,110, + 32,114,101,112,114,101,115,101,110,116,115,32,116,104,101,32, + 103,114,101,97,116,101,115,116,32,99,111,109,109,111,110,32, + 100,101,110,111,109,105,110,97,116,111,114,32,111,102,32,102, + 117,110,99,116,105,111,110,97,108,105,116,121,10,32,32,32, + 32,98,101,116,119,101,101,110,32,105,109,112,111,114,116,95, + 109,111,100,117,108,101,32,97,110,100,32,95,95,105,109,112, + 111,114,116,95,95,46,32,84,104,105,115,32,105,110,99,108, + 117,100,101,115,32,115,101,116,116,105,110,103,32,95,95,112, + 97,99,107,97,103,101,95,95,32,105,102,10,32,32,32,32, + 116,104,101,32,108,111,97,100,101,114,32,100,105,100,32,110, + 111,116,46,10,10,32,32,32,32,114,33,0,0,0,78,122, + 40,105,109,112,111,114,116,32,111,102,32,123,125,32,104,97, + 108,116,101,100,59,32,78,111,110,101,32,105,110,32,115,121, + 115,46,109,111,100,117,108,101,115,114,15,0,0,0,41,12, + 114,182,0,0,0,114,172,0,0,0,114,57,0,0,0,114, + 146,0,0,0,114,14,0,0,0,114,21,0,0,0,114,186, + 0,0,0,218,11,95,103,99,100,95,105,109,112,111,114,116, + 114,58,0,0,0,114,50,0,0,0,114,77,0,0,0,114, + 63,0,0,0,41,5,114,15,0,0,0,114,170,0,0,0, + 114,171,0,0,0,114,89,0,0,0,114,74,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,114,187, + 0,0,0,214,3,0,0,115,28,0,0,0,0,9,16,1, + 12,1,18,1,10,1,15,1,13,1,13,1,12,1,10,1, + 6,1,9,1,18,1,10,1,114,187,0,0,0,99,3,0, + 0,0,0,0,0,0,6,0,0,0,17,0,0,0,67,0, + 0,0,115,239,0,0,0,116,0,0,124,0,0,100,1,0, + 131,2,0,114,235,0,100,2,0,124,1,0,107,6,0,114, + 83,0,116,1,0,124,1,0,131,1,0,125,1,0,124,1, + 0,106,2,0,100,2,0,131,1,0,1,116,0,0,124,0, + 0,100,3,0,131,2,0,114,83,0,124,1,0,106,3,0, + 124,0,0,106,4,0,131,1,0,1,120,149,0,124,1,0, + 68,93,141,0,125,3,0,116,0,0,124,0,0,124,3,0, + 131,2,0,115,90,0,100,4,0,106,5,0,124,0,0,106, + 6,0,124,3,0,131,2,0,125,4,0,121,17,0,116,7, + 0,124,2,0,124,4,0,131,2,0,1,87,113,90,0,4, + 116,8,0,107,10,0,114,230,0,1,125,5,0,1,122,47, + 0,116,9,0,124,5,0,131,1,0,106,10,0,116,11,0, + 131,1,0,114,209,0,124,5,0,106,12,0,124,4,0,107, + 2,0,114,209,0,119,90,0,130,0,0,87,89,100,5,0, + 100,5,0,125,5,0,126,5,0,88,113,90,0,88,113,90, + 0,87,124,0,0,83,41,6,122,238,70,105,103,117,114,101, + 32,111,117,116,32,119,104,97,116,32,95,95,105,109,112,111, + 114,116,95,95,32,115,104,111,117,108,100,32,114,101,116,117, + 114,110,46,10,10,32,32,32,32,84,104,101,32,105,109,112, + 111,114,116,95,32,112,97,114,97,109,101,116,101,114,32,105, + 115,32,97,32,99,97,108,108,97,98,108,101,32,119,104,105, + 99,104,32,116,97,107,101,115,32,116,104,101,32,110,97,109, + 101,32,111,102,32,109,111,100,117,108,101,32,116,111,10,32, + 32,32,32,105,109,112,111,114,116,46,32,73,116,32,105,115, + 32,114,101,113,117,105,114,101,100,32,116,111,32,100,101,99, + 111,117,112,108,101,32,116,104,101,32,102,117,110,99,116,105, + 111,110,32,102,114,111,109,32,97,115,115,117,109,105,110,103, + 32,105,109,112,111,114,116,108,105,98,39,115,10,32,32,32, + 32,105,109,112,111,114,116,32,105,109,112,108,101,109,101,110, + 116,97,116,105,111,110,32,105,115,32,100,101,115,105,114,101, + 100,46,10,10,32,32,32,32,114,131,0,0,0,250,1,42, + 218,7,95,95,97,108,108,95,95,122,5,123,125,46,123,125, + 78,41,13,114,4,0,0,0,114,130,0,0,0,218,6,114, + 101,109,111,118,101,218,6,101,120,116,101,110,100,114,189,0, + 0,0,114,50,0,0,0,114,1,0,0,0,114,65,0,0, + 0,114,77,0,0,0,114,179,0,0,0,114,71,0,0,0, + 218,15,95,69,82,82,95,77,83,71,95,80,82,69,70,73, + 88,114,15,0,0,0,41,6,114,89,0,0,0,218,8,102, + 114,111,109,108,105,115,116,114,184,0,0,0,218,1,120,90, + 9,102,114,111,109,95,110,97,109,101,90,3,101,120,99,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,16, + 95,104,97,110,100,108,101,95,102,114,111,109,108,105,115,116, + 238,3,0,0,115,34,0,0,0,0,10,15,1,12,1,12, + 1,13,1,15,1,16,1,13,1,15,1,21,1,3,1,17, + 1,18,4,21,1,15,1,3,1,26,1,114,195,0,0,0, + 99,1,0,0,0,0,0,0,0,3,0,0,0,7,0,0, + 0,67,0,0,0,115,214,0,0,0,124,0,0,106,0,0, + 100,1,0,131,1,0,125,1,0,124,0,0,106,0,0,100, + 2,0,131,1,0,125,2,0,124,1,0,100,3,0,107,9, + 0,114,128,0,124,2,0,100,3,0,107,9,0,114,124,0, + 124,1,0,124,2,0,106,1,0,107,3,0,114,124,0,116, + 2,0,106,3,0,100,4,0,106,4,0,100,5,0,124,1, + 0,155,2,0,100,6,0,124,2,0,106,1,0,155,2,0, + 100,7,0,103,5,0,131,1,0,116,5,0,100,8,0,100, + 9,0,131,2,1,1,124,1,0,83,124,2,0,100,3,0, + 107,9,0,114,147,0,124,2,0,106,1,0,83,116,2,0, + 106,3,0,100,10,0,116,5,0,100,8,0,100,9,0,131, + 2,1,1,124,0,0,100,11,0,25,125,1,0,100,12,0, + 124,0,0,107,7,0,114,210,0,124,1,0,106,6,0,100, + 13,0,131,1,0,100,14,0,25,125,1,0,124,1,0,83, + 41,15,122,167,67,97,108,99,117,108,97,116,101,32,119,104, + 97,116,32,95,95,112,97,99,107,97,103,101,95,95,32,115, + 104,111,117,108,100,32,98,101,46,10,10,32,32,32,32,95, + 95,112,97,99,107,97,103,101,95,95,32,105,115,32,110,111, + 116,32,103,117,97,114,97,110,116,101,101,100,32,116,111,32, + 98,101,32,100,101,102,105,110,101,100,32,111,114,32,99,111, + 117,108,100,32,98,101,32,115,101,116,32,116,111,32,78,111, + 110,101,10,32,32,32,32,116,111,32,114,101,112,114,101,115, + 101,110,116,32,116,104,97,116,32,105,116,115,32,112,114,111, + 112,101,114,32,118,97,108,117,101,32,105,115,32,117,110,107, + 110,111,119,110,46,10,10,32,32,32,32,114,134,0,0,0, + 114,95,0,0,0,78,218,0,122,32,95,95,112,97,99,107, + 97,103,101,95,95,32,33,61,32,95,95,115,112,101,99,95, + 95,46,112,97,114,101,110,116,32,40,122,4,32,33,61,32, + 250,1,41,114,140,0,0,0,233,3,0,0,0,122,89,99, + 97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99, + 107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99, + 95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95, + 95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32, + 111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32, + 95,95,112,97,116,104,95,95,114,1,0,0,0,114,131,0, + 0,0,114,121,0,0,0,114,33,0,0,0,41,7,114,42, + 0,0,0,114,123,0,0,0,114,142,0,0,0,114,143,0, + 0,0,114,115,0,0,0,114,176,0,0,0,114,122,0,0, + 0,41,3,218,7,103,108,111,98,97,108,115,114,170,0,0, + 0,114,88,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,17,95,99,97,108,99,95,95,95,112, + 97,99,107,97,103,101,95,95,14,4,0,0,115,30,0,0, + 0,0,7,15,1,15,1,12,1,27,1,42,2,13,1,4, + 1,12,1,7,2,9,2,13,1,10,1,12,1,19,1,114, + 200,0,0,0,99,5,0,0,0,0,0,0,0,9,0,0, + 0,5,0,0,0,67,0,0,0,115,227,0,0,0,124,4, + 0,100,1,0,107,2,0,114,27,0,116,0,0,124,0,0, + 131,1,0,125,5,0,110,54,0,124,1,0,100,2,0,107, + 9,0,114,45,0,124,1,0,110,3,0,105,0,0,125,6, + 0,116,1,0,124,6,0,131,1,0,125,7,0,116,0,0, + 124,0,0,124,7,0,124,4,0,131,3,0,125,5,0,124, + 3,0,115,207,0,124,4,0,100,1,0,107,2,0,114,122, + 0,116,0,0,124,0,0,106,2,0,100,3,0,131,1,0, + 100,1,0,25,131,1,0,83,124,0,0,115,132,0,124,5, + 0,83,116,3,0,124,0,0,131,1,0,116,3,0,124,0, + 0,106,2,0,100,3,0,131,1,0,100,1,0,25,131,1, + 0,24,125,8,0,116,4,0,106,5,0,124,5,0,106,6, + 0,100,2,0,116,3,0,124,5,0,106,6,0,131,1,0, + 124,8,0,24,133,2,0,25,25,83,110,16,0,116,7,0, + 124,5,0,124,3,0,116,0,0,131,3,0,83,100,2,0, + 83,41,4,97,214,1,0,0,73,109,112,111,114,116,32,97, + 32,109,111,100,117,108,101,46,10,10,32,32,32,32,84,104, + 101,32,39,103,108,111,98,97,108,115,39,32,97,114,103,117, + 109,101,110,116,32,105,115,32,117,115,101,100,32,116,111,32, + 105,110,102,101,114,32,119,104,101,114,101,32,116,104,101,32, + 105,109,112,111,114,116,32,105,115,32,111,99,99,117,114,105, + 110,103,32,102,114,111,109,10,32,32,32,32,116,111,32,104, + 97,110,100,108,101,32,114,101,108,97,116,105,118,101,32,105, + 109,112,111,114,116,115,46,32,84,104,101,32,39,108,111,99, + 97,108,115,39,32,97,114,103,117,109,101,110,116,32,105,115, + 32,105,103,110,111,114,101,100,46,32,84,104,101,10,32,32, + 32,32,39,102,114,111,109,108,105,115,116,39,32,97,114,103, + 117,109,101,110,116,32,115,112,101,99,105,102,105,101,115,32, + 119,104,97,116,32,115,104,111,117,108,100,32,101,120,105,115, + 116,32,97,115,32,97,116,116,114,105,98,117,116,101,115,32, + 111,110,32,116,104,101,32,109,111,100,117,108,101,10,32,32, + 32,32,98,101,105,110,103,32,105,109,112,111,114,116,101,100, + 32,40,101,46,103,46,32,96,96,102,114,111,109,32,109,111, + 100,117,108,101,32,105,109,112,111,114,116,32,60,102,114,111, + 109,108,105,115,116,62,96,96,41,46,32,32,84,104,101,32, + 39,108,101,118,101,108,39,10,32,32,32,32,97,114,103,117, + 109,101,110,116,32,114,101,112,114,101,115,101,110,116,115,32, + 116,104,101,32,112,97,99,107,97,103,101,32,108,111,99,97, + 116,105,111,110,32,116,111,32,105,109,112,111,114,116,32,102, + 114,111,109,32,105,110,32,97,32,114,101,108,97,116,105,118, + 101,10,32,32,32,32,105,109,112,111,114,116,32,40,101,46, + 103,46,32,96,96,102,114,111,109,32,46,46,112,107,103,32, + 105,109,112,111,114,116,32,109,111,100,96,96,32,119,111,117, + 108,100,32,104,97,118,101,32,97,32,39,108,101,118,101,108, + 39,32,111,102,32,50,41,46,10,10,32,32,32,32,114,33, + 0,0,0,78,114,121,0,0,0,41,8,114,187,0,0,0, + 114,200,0,0,0,218,9,112,97,114,116,105,116,105,111,110, + 114,168,0,0,0,114,14,0,0,0,114,21,0,0,0,114, + 1,0,0,0,114,195,0,0,0,41,9,114,15,0,0,0, + 114,199,0,0,0,218,6,108,111,99,97,108,115,114,193,0, + 0,0,114,171,0,0,0,114,89,0,0,0,90,8,103,108, + 111,98,97,108,115,95,114,170,0,0,0,90,7,99,117,116, + 95,111,102,102,114,10,0,0,0,114,10,0,0,0,114,11, + 0,0,0,218,10,95,95,105,109,112,111,114,116,95,95,41, + 4,0,0,115,26,0,0,0,0,11,12,1,15,2,24,1, + 12,1,18,1,6,3,12,1,23,1,6,1,4,4,35,3, + 40,2,114,203,0,0,0,99,1,0,0,0,0,0,0,0, + 2,0,0,0,3,0,0,0,67,0,0,0,115,53,0,0, + 0,116,0,0,106,1,0,124,0,0,131,1,0,125,1,0, + 124,1,0,100,0,0,107,8,0,114,43,0,116,2,0,100, + 1,0,124,0,0,23,131,1,0,130,1,0,116,3,0,124, + 1,0,131,1,0,83,41,2,78,122,25,110,111,32,98,117, + 105,108,116,45,105,110,32,109,111,100,117,108,101,32,110,97, + 109,101,100,32,41,4,114,151,0,0,0,114,155,0,0,0, + 114,77,0,0,0,114,150,0,0,0,41,2,114,15,0,0, + 0,114,88,0,0,0,114,10,0,0,0,114,10,0,0,0, + 114,11,0,0,0,218,18,95,98,117,105,108,116,105,110,95, + 102,114,111,109,95,110,97,109,101,76,4,0,0,115,8,0, + 0,0,0,1,15,1,12,1,16,1,114,204,0,0,0,99, + 2,0,0,0,0,0,0,0,12,0,0,0,12,0,0,0, + 67,0,0,0,115,74,1,0,0,124,1,0,97,0,0,124, + 0,0,97,1,0,116,2,0,116,1,0,131,1,0,125,2, + 0,120,123,0,116,1,0,106,3,0,106,4,0,131,0,0, + 68,93,106,0,92,2,0,125,3,0,125,4,0,116,5,0, + 124,4,0,124,2,0,131,2,0,114,40,0,124,3,0,116, + 1,0,106,6,0,107,6,0,114,91,0,116,7,0,125,5, + 0,110,27,0,116,0,0,106,8,0,124,3,0,131,1,0, + 114,40,0,116,9,0,125,5,0,110,3,0,113,40,0,116, + 10,0,124,4,0,124,5,0,131,2,0,125,6,0,116,11, + 0,124,6,0,124,4,0,131,2,0,1,113,40,0,87,116, + 1,0,106,3,0,116,12,0,25,125,7,0,120,73,0,100, + 5,0,68,93,65,0,125,8,0,124,8,0,116,1,0,106, + 3,0,107,7,0,114,206,0,116,13,0,124,8,0,131,1, + 0,125,9,0,110,13,0,116,1,0,106,3,0,124,8,0, + 25,125,9,0,116,14,0,124,7,0,124,8,0,124,9,0, + 131,3,0,1,113,170,0,87,121,16,0,116,13,0,100,2, + 0,131,1,0,125,10,0,87,110,24,0,4,116,15,0,107, + 10,0,114,25,1,1,1,1,100,3,0,125,10,0,89,110, + 1,0,88,116,14,0,124,7,0,100,2,0,124,10,0,131, + 3,0,1,116,13,0,100,4,0,131,1,0,125,11,0,116, + 14,0,124,7,0,100,4,0,124,11,0,131,3,0,1,100, + 3,0,83,41,6,122,250,83,101,116,117,112,32,105,109,112, + 111,114,116,108,105,98,32,98,121,32,105,109,112,111,114,116, + 105,110,103,32,110,101,101,100,101,100,32,98,117,105,108,116, + 45,105,110,32,109,111,100,117,108,101,115,32,97,110,100,32, + 105,110,106,101,99,116,105,110,103,32,116,104,101,109,10,32, + 32,32,32,105,110,116,111,32,116,104,101,32,103,108,111,98, + 97,108,32,110,97,109,101,115,112,97,99,101,46,10,10,32, + 32,32,32,65,115,32,115,121,115,32,105,115,32,110,101,101, + 100,101,100,32,102,111,114,32,115,121,115,46,109,111,100,117, + 108,101,115,32,97,99,99,101,115,115,32,97,110,100,32,95, + 105,109,112,32,105,115,32,110,101,101,100,101,100,32,116,111, + 32,108,111,97,100,32,98,117,105,108,116,45,105,110,10,32, + 32,32,32,109,111,100,117,108,101,115,44,32,116,104,111,115, + 101,32,116,119,111,32,109,111,100,117,108,101,115,32,109,117, + 115,116,32,98,101,32,101,120,112,108,105,99,105,116,108,121, + 32,112,97,115,115,101,100,32,105,110,46,10,10,32,32,32, + 32,114,142,0,0,0,114,34,0,0,0,78,114,62,0,0, + 0,41,1,122,9,95,119,97,114,110,105,110,103,115,41,16, + 114,57,0,0,0,114,14,0,0,0,114,13,0,0,0,114, + 21,0,0,0,218,5,105,116,101,109,115,114,178,0,0,0, + 114,76,0,0,0,114,151,0,0,0,114,82,0,0,0,114, + 161,0,0,0,114,132,0,0,0,114,137,0,0,0,114,1, + 0,0,0,114,204,0,0,0,114,5,0,0,0,114,77,0, + 0,0,41,12,218,10,115,121,115,95,109,111,100,117,108,101, + 218,11,95,105,109,112,95,109,111,100,117,108,101,90,11,109, + 111,100,117,108,101,95,116,121,112,101,114,15,0,0,0,114, + 89,0,0,0,114,99,0,0,0,114,88,0,0,0,90,11, + 115,101,108,102,95,109,111,100,117,108,101,90,12,98,117,105, + 108,116,105,110,95,110,97,109,101,90,14,98,117,105,108,116, + 105,110,95,109,111,100,117,108,101,90,13,116,104,114,101,97, + 100,95,109,111,100,117,108,101,90,14,119,101,97,107,114,101, + 102,95,109,111,100,117,108,101,114,10,0,0,0,114,10,0, + 0,0,114,11,0,0,0,218,6,95,115,101,116,117,112,83, + 4,0,0,115,50,0,0,0,0,9,6,1,6,3,12,1, + 28,1,15,1,15,1,9,1,15,1,9,2,3,1,15,1, + 17,3,13,1,13,1,15,1,15,2,13,1,20,3,3,1, + 16,1,13,2,11,1,16,3,12,1,114,208,0,0,0,99, + 2,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0, + 67,0,0,0,115,87,0,0,0,116,0,0,124,0,0,124, + 1,0,131,2,0,1,116,1,0,106,2,0,106,3,0,116, + 4,0,131,1,0,1,116,1,0,106,2,0,106,3,0,116, + 5,0,131,1,0,1,100,1,0,100,2,0,108,6,0,125, + 2,0,124,2,0,97,7,0,124,2,0,106,8,0,116,1, + 0,106,9,0,116,10,0,25,131,1,0,1,100,2,0,83, + 41,3,122,50,73,110,115,116,97,108,108,32,105,109,112,111, + 114,116,108,105,98,32,97,115,32,116,104,101,32,105,109,112, + 108,101,109,101,110,116,97,116,105,111,110,32,111,102,32,105, + 109,112,111,114,116,46,114,33,0,0,0,78,41,11,114,208, + 0,0,0,114,14,0,0,0,114,175,0,0,0,114,113,0, + 0,0,114,151,0,0,0,114,161,0,0,0,218,26,95,102, + 114,111,122,101,110,95,105,109,112,111,114,116,108,105,98,95, + 101,120,116,101,114,110,97,108,114,119,0,0,0,218,8,95, + 105,110,115,116,97,108,108,114,21,0,0,0,114,1,0,0, + 0,41,3,114,206,0,0,0,114,207,0,0,0,114,209,0, + 0,0,114,10,0,0,0,114,10,0,0,0,114,11,0,0, + 0,114,210,0,0,0,130,4,0,0,115,12,0,0,0,0, + 2,13,2,16,1,16,3,12,1,6,1,114,210,0,0,0, + 41,51,114,3,0,0,0,114,119,0,0,0,114,12,0,0, + 0,114,16,0,0,0,114,17,0,0,0,114,59,0,0,0, + 114,41,0,0,0,114,48,0,0,0,114,31,0,0,0,114, + 32,0,0,0,114,53,0,0,0,114,54,0,0,0,114,56, + 0,0,0,114,63,0,0,0,114,65,0,0,0,114,75,0, + 0,0,114,81,0,0,0,114,84,0,0,0,114,90,0,0, + 0,114,101,0,0,0,114,102,0,0,0,114,106,0,0,0, + 114,85,0,0,0,218,6,111,98,106,101,99,116,90,9,95, + 80,79,80,85,76,65,84,69,114,132,0,0,0,114,137,0, + 0,0,114,145,0,0,0,114,97,0,0,0,114,86,0,0, + 0,114,149,0,0,0,114,150,0,0,0,114,87,0,0,0, + 114,151,0,0,0,114,161,0,0,0,114,166,0,0,0,114, + 172,0,0,0,114,174,0,0,0,114,177,0,0,0,114,182, + 0,0,0,114,192,0,0,0,114,183,0,0,0,114,185,0, + 0,0,114,186,0,0,0,114,187,0,0,0,114,195,0,0, + 0,114,200,0,0,0,114,203,0,0,0,114,204,0,0,0, + 114,208,0,0,0,114,210,0,0,0,114,10,0,0,0,114, + 10,0,0,0,114,10,0,0,0,114,11,0,0,0,218,8, + 60,109,111,100,117,108,101,62,8,0,0,0,115,96,0,0, + 0,6,17,6,2,12,8,12,4,19,20,6,2,6,3,22, + 4,19,68,19,21,19,19,12,19,12,19,12,11,18,8,12, + 11,12,12,12,16,12,36,19,27,19,101,24,26,9,3,18, + 45,18,60,12,18,12,17,12,25,12,29,12,23,12,16,19, + 73,19,77,19,13,12,9,12,9,15,47,12,20,6,1,10, + 2,12,27,12,6,18,24,12,32,12,27,24,35,12,7,12, + 47, }; -- cgit v0.12 From bfba2cd40629e52ba74ddc75a3415adc8f0ef71b Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 24 Mar 2016 22:44:41 -0500 Subject: Don't doc check the venv dir --- Doc/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/Makefile b/Doc/Makefile index a42e98b..03a37f1 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -153,7 +153,7 @@ dist: cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub check: - $(PYTHON) tools/rstlint.py -i tools + $(PYTHON) tools/rstlint.py -i tools -i venv serve: ../Tools/scripts/serve.py build/html -- cgit v0.12 From 4a2e663f9941799eeeda5a0eff320665ae1d8e8b Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 24 Mar 2016 22:45:00 -0500 Subject: Fix a few typos --- Misc/NEWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS index d684c82..945527c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,10 +28,10 @@ Core and Builtins :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. -- Issue #26516: Add :envvar`PYTHONMALLOC` environment variable to set the +- Issue #26516: Add :envvar:`PYTHONMALLOC` environment variable to set the Python memory allocators and/or install debug hooks. -- Issue #26516: The :c:func`PyMem_SetupDebugHooks` function can now also be +- Issue #26516: The :c:func:`PyMem_SetupDebugHooks` function can now also be used on Python compiled in release mode. - Issue #26516: The :envvar:`PYTHONMALLOCSTATS` environment variable can now @@ -238,7 +238,7 @@ Library ``None``). - Issue #21925: :func:`warnings.formatwarning` now catches exceptions when - calling :func;`linecache.getline` and + calling :func:`linecache.getline` and :func:`tracemalloc.get_object_traceback` to be able to log :exc:`ResourceWarning` emitted late during the Python shutdown process. -- cgit v0.12 From 7285d520e00e110af2f150487fc0ccfaa8430684 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 24 Mar 2016 22:43:23 -0700 Subject: remove duplicated check for fractions and complex numbers (closes #26076) Patch by Oren Milman. --- Parser/tokenizer.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index c653121..006ad9c 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1587,10 +1587,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) if (c == '0') { /* Hex, octal or binary -- maybe. */ c = tok_nextc(tok); - if (c == '.') - goto fraction; - if (c == 'j' || c == 'J') - goto imaginary; if (c == 'x' || c == 'X') { /* Hex */ -- cgit v0.12 From 633ebda3ba57428835d27866d8567af1c857fcd2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 08:57:16 +0100 Subject: Issue #26637: Fix test_io The import machinery now raises a different exception when it fails at Python shutdown. --- Lib/test/test_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 86440c5..37b9b2d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3079,8 +3079,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio - #shutdown_error = "LookupError: unknown encoding: ascii" - shutdown_error = "TypeError: 'NoneType' object is not iterable" + shutdown_error = "LookupError: unknown encoding: ascii" class IncrementalNewlineDecoderTest(unittest.TestCase): -- cgit v0.12 From 47b4557679154a645780b2e44dd7a4a2c62b5d12 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:07:07 +0100 Subject: test_io: ignore DeprecationWarning on bytes path on Windows --- Lib/test/test_io.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 37b9b2d..db074ca 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -364,7 +364,11 @@ class IOTest(unittest.TestCase): def test_open_handles_NUL_chars(self): fn_with_NUL = 'foo\0bar' self.assertRaises(ValueError, self.open, fn_with_NUL, 'w') - self.assertRaises(ValueError, self.open, bytes(fn_with_NUL, 'ascii'), 'w') + + bytes_fn = bytes(fn_with_NUL, 'ascii') + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertRaises(ValueError, self.open, bytes_fn, 'w') def test_raw_file_io(self): with self.open(support.TESTFN, "wb", buffering=0) as f: -- cgit v0.12 From a6d865c128dd46a067358e94c29ca2d84205ae89 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:29:50 +0100 Subject: Issue #25654: * multiprocessing: open file with closefd=False to avoid ResourceWarning * _test_multiprocessing: open file with O_EXCL to detect bugs in tests (if a previous test forgot to remove TESTFN) * test_sys_exit(): remove TESTFN after each loop iteration Initial patch written by Serhiy Storchaka. --- Doc/library/multiprocessing.rst | 2 +- Lib/multiprocessing/forkserver.py | 8 +------- Lib/multiprocessing/process.py | 7 +------ Lib/multiprocessing/util.py | 23 +++++++++++++++++++++++ Lib/test/_test_multiprocessing.py | 18 +++++++++++++----- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index bf25f3f..6440f5c 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -2694,7 +2694,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" in issues with processes-in-processes. This has been changed to:: sys.stdin.close() - sys.stdin = open(os.devnull) + sys.stdin = open(os.open(os.devnull, os.O_RDONLY), closefd=False) Which solves the fundamental issue of processes colliding with each other resulting in a bad file descriptor error, but introduces a potential danger diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index b27cba5..ad01ede 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -147,13 +147,7 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): except ImportError: pass - # close sys.stdin - if sys.stdin is not None: - try: - sys.stdin.close() - sys.stdin = open(os.devnull) - except (OSError, ValueError): - pass + util._close_stdin() # ignoring SIGCHLD means no need to reap zombie processes handler = signal.signal(signal.SIGCHLD, signal.SIG_IGN) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 68959bf..bca8b7a 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -234,12 +234,7 @@ class BaseProcess(object): context._force_start_method(self._start_method) _process_counter = itertools.count(1) _children = set() - if sys.stdin is not None: - try: - sys.stdin.close() - sys.stdin = open(os.devnull) - except (OSError, ValueError): - pass + util._close_stdin() old_process = _current_process _current_process = self try: diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index ea5443d..1a2c0db 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -9,6 +9,7 @@ import os import itertools +import sys import weakref import atexit import threading # we want threading to install it's @@ -356,6 +357,28 @@ def close_all_fds_except(fds): assert fds[-1] == MAXFD, 'fd too large' for i in range(len(fds) - 1): os.closerange(fds[i]+1, fds[i+1]) +# +# Close sys.stdin and replace stdin with os.devnull +# + +def _close_stdin(): + if sys.stdin is None: + return + + try: + sys.stdin.close() + except (OSError, ValueError): + pass + + try: + fd = os.open(os.devnull, os.O_RDONLY) + try: + sys.stdin = open(fd, closefd=False) + except: + os.close(fd) + raise + except (OSError, ValueError): + pass # # Start a program with only specified fds kept open diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 95d418d..16407db 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -455,13 +455,15 @@ class _TestSubclassingProcess(BaseTestCase): @classmethod def _test_stderr_flush(cls, testfn): - sys.stderr = open(testfn, 'w') + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', closefd=False) 1/0 # MARKER @classmethod def _test_sys_exit(cls, reason, testfn): - sys.stderr = open(testfn, 'w') + fd = os.open(testfn, os.O_WRONLY | os.O_CREAT | os.O_EXCL) + sys.stderr = open(fd, 'w', closefd=False) sys.exit(reason) def test_sys_exit(self): @@ -472,15 +474,21 @@ class _TestSubclassingProcess(BaseTestCase): testfn = test.support.TESTFN self.addCleanup(test.support.unlink, testfn) - for reason, code in (([1, 2, 3], 1), ('ignore this', 1)): + for reason in ( + [1, 2, 3], + 'ignore this', + ): p = self.Process(target=self._test_sys_exit, args=(reason, testfn)) p.daemon = True p.start() p.join(5) - self.assertEqual(p.exitcode, code) + self.assertEqual(p.exitcode, 1) with open(testfn, 'r') as f: - self.assertEqual(f.read().rstrip(), str(reason)) + content = f.read() + self.assertEqual(content.rstrip(), str(reason)) + + os.unlink(testfn) for reason in (True, False, 8): p = self.Process(target=sys.exit, args=(reason,)) -- cgit v0.12 From 6c45d397a32a1f887dc5ed1565b57bf570ded1a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 09:51:14 +0100 Subject: Issue #21925: Fix test_warnings for release mode Use -Wd comment line option to log the ResourceWarning. --- Lib/test/test_warnings/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 9f1cd75..42c7603 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1007,12 +1007,12 @@ a=A() # don't import the warnings module # (_warnings will try to import it) code = "f = open(%a)" % __file__ - rc, out, err = assert_python_ok("-c", code) + rc, out, err = assert_python_ok("-Wd", "-c", code) self.assertTrue(err.startswith(expected), ascii(err)) # import the warnings module code = "import warnings; f = open(%a)" % __file__ - rc, out, err = assert_python_ok("-c", code) + rc, out, err = assert_python_ok("-Wd", "-c", code) self.assertTrue(err.startswith(expected), ascii(err)) -- cgit v0.12 From b72e21b9ab8021b7cbb7e0007d1bf80d533bce15 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 25 Mar 2016 02:29:59 -0700 Subject: Speed-up construction of empty sets by approx 12-14%. --- Objects/setobject.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index c9834a8..8e22b69 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2015,11 +2015,12 @@ set_init(PySetObject *self, PyObject *args, PyObject *kwds) if (!PyAnySet_Check(self)) return -1; - if (PySet_Check(self) && !_PyArg_NoKeywords("set()", kwds)) + if (kwds != NULL && PySet_Check(self) && !_PyArg_NoKeywords("set()", kwds)) return -1; if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; - set_clear_internal(self); + if (self->fill) + set_clear_internal(self); self->hash = -1; if (iterable == NULL) return 0; -- cgit v0.12 From e77c974357fa96917e6966d627a69eae8a919b37 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 10:28:23 +0100 Subject: test_os: Win32ErrorTests now ensures that TESTFN doesn't exist Replace also other open(filename, "w") with open(filename, "x") to fail if a previous test forgot to remove filename. --- Lib/test/test_os.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bc24e47..c8789d7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -845,9 +845,8 @@ class WalkTests(unittest.TestCase): os.makedirs(t2_path) for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path: - f = open(path, "w") - f.write("I'm " + path + " and proud of it. Blame test_os.\n") - f.close() + with open(path, "x") as f: + f.write("I'm " + path + " and proud of it. Blame test_os.\n") if support.can_symlink(): os.symlink(os.path.abspath(t2_path), self.link_path) @@ -1427,6 +1426,9 @@ class ExecTests(unittest.TestCase): @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): + def setUp(self): + self.assertFalse(os.path.exists(support.TESTFN)) + def test_rename(self): self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") @@ -1439,7 +1441,7 @@ class Win32ErrorTests(unittest.TestCase): def test_mkdir(self): self.addCleanup(support.unlink, support.TESTFN) - with open(support.TESTFN, "w") as f: + with open(support.TESTFN, "x") as f: self.assertRaises(OSError, os.mkdir, support.TESTFN) def test_utime(self): @@ -1448,6 +1450,7 @@ class Win32ErrorTests(unittest.TestCase): def test_chmod(self): self.assertRaises(OSError, os.chmod, support.TESTFN, 0) + class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] @@ -1559,8 +1562,7 @@ class LinkTests(unittest.TestCase): os.unlink(file) def _test_link(self, file1, file2): - with open(file1, "w") as f1: - f1.write("test") + create_file(file1) with bytes_filename_warn(False): os.link(file1, file2) -- cgit v0.12 From a9a852c2b123031febf245d3bd37a529e0635d01 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 11:54:47 +0100 Subject: Modernize Python/makeopcodetargets.py * Simply use "import opcode" to import the opcode module instead of tricks using the imp module * Use context manager for the output file * Move code into a new main() function * Replace assert with a regular if to check the number of arguments * Import modules at top level --- Python/makeopcodetargets.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index d9a0855..7a57a06 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,24 +3,14 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ -# This code should stay compatible with Python 2.3, at least while -# some of the buildbots have Python 2.3 as their system Python. - -import imp +import opcode import os +import sys -def find_module(modname): - """Finds and returns a module in the local dist/checkout. - """ - modpath = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "Lib") - return imp.load_module(modname, *imp.find_module(modname, [modpath])) - def write_contents(f): """Write C code contents to the target file object. """ - opcode = find_module("opcode") targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname @@ -29,15 +19,17 @@ def write_contents(f): f.write("\n};\n") -if __name__ == "__main__": - import sys - assert len(sys.argv) < 3, "Too many arguments" +def main(): + if len(sys.argv) >= 3: + sys.exit("Too many arguments") if len(sys.argv) == 2: target = sys.argv[1] else: target = "Python/opcode_targets.h" - f = open(target, "w") - try: + with open(target, "w") as f: write_contents(f) - finally: - f.close() + print("Jump table written into %s" % target) + + +if __name__ == "__main__": + main() -- cgit v0.12 From b347788b825feb156510d6ec321930c6044bd202 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:27:02 +0100 Subject: Skip test_venv.test_with_pip() if ctypes miss Issue #26610. --- Lib/test/test_venv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index dbb8b85..09741e3 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -31,6 +31,11 @@ try: except ImportError: threading = None +try: + import ctypes +except ImportError: + ctypes = None + skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate in a venv') @@ -327,6 +332,8 @@ class EnsurePipTest(BaseTest): @unittest.skipIf(ssl is None, ensurepip._MISSING_SSL_MESSAGE) @unittest.skipUnless(threading, 'some dependencies of pip import threading' ' module unconditionally') + # Issue #26610: pip/pep425tags.py requires ctypes + @unittest.skipUnless(ctypes, 'pip requires ctypes') def test_with_pip(self): rmtree(self.env_dir) with EnvironmentVarGuard() as envvars: -- cgit v0.12 From bdc337b7a8bbe5719cf7aeab7e104262fdd741f0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:30:40 +0100 Subject: test_venv: enhance test_devnull() --- Lib/test/test_venv.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 09741e3..55bee23 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -39,15 +39,9 @@ except ImportError: skipInVenv = unittest.skipIf(sys.prefix != sys.base_prefix, 'Test not appropriate in a venv') -# os.path.exists('nul') is False: http://bugs.python.org/issue20541 -if os.devnull.lower() == 'nul': - failsOnWindows = unittest.expectedFailure -else: - def failsOnWindows(f): - return f - class BaseTest(unittest.TestCase): """Base class for venv tests.""" + maxDiff = 80 * 50 def setUp(self): self.env_dir = os.path.realpath(tempfile.mkdtemp()) @@ -318,16 +312,21 @@ class EnsurePipTest(BaseTest): self.run_with_capture(venv.create, self.env_dir, with_pip=False) self.assert_pip_not_installed() - @failsOnWindows - def test_devnull_exists_and_is_empty(self): + def test_devnull(self): # Fix for issue #20053 uses os.devnull to force a config file to # appear empty. However http://bugs.python.org/issue20541 means # that doesn't currently work properly on Windows. Once that is # fixed, the "win_location" part of test_with_pip should be restored - self.assertTrue(os.path.exists(os.devnull)) with open(os.devnull, "rb") as f: self.assertEqual(f.read(), b"") + # Issue #20541: os.path.exists('nul') is False on Windows + if os.devnull.lower() == 'nul': + self.assertFalse(os.path.exists(os.devnull)) + else: + self.assertTrue(os.path.exists(os.devnull)) + + # Requesting pip fails without SSL (http://bugs.python.org/issue19744) @unittest.skipIf(ssl is None, ensurepip._MISSING_SSL_MESSAGE) @unittest.skipUnless(threading, 'some dependencies of pip import threading' -- cgit v0.12 From 931602a1acea08d5288c7d8d99996082d0e0aa8b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 12:48:17 +0100 Subject: test_doctest: remove unused imports --- Lib/test/test_doctest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 6a7ba0b..f7c4806 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -2948,12 +2948,11 @@ Invalid doctest option: def test_main(): # Check the doctest cases in doctest itself: ret = support.run_doctest(doctest, verbosity=True) + # Check the doctest cases defined here: from test import test_doctest support.run_doctest(test_doctest, verbosity=True) -import sys, re, io - def test_coverage(coverdir): trace = support.import_module('trace') tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,], -- cgit v0.12 From 32830149d8873234eaf1949ef840c3a07ecf5b64 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 15:12:08 +0100 Subject: changeset: 100749:0b61b2d28a07 tag: tip parent: 100742:ebae81b31cf6 user: Victor Stinner date: Fri Mar 25 15:03:34 2016 +0100 files: Lib/test/test_os.py description: test_os: Win32ErrorTests checks if file exists Don't use os.path.exists() since it ignores *any* OSError. --- Lib/test/test_os.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c8789d7..f7d64b7 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1427,7 +1427,16 @@ class ExecTests(unittest.TestCase): @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") class Win32ErrorTests(unittest.TestCase): def setUp(self): - self.assertFalse(os.path.exists(support.TESTFN)) + try: + os.stat(support.TESTFN) + except FileNotFoundError: + exists = False + except OSError as exc: + exists = True + self.fail("file %s must not exist; os.stat failed with %s" + % (support.TESTFN, exc)) + else: + self.fail("file %s must not exist" % support.TESTFN) def test_rename(self): self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") -- cgit v0.12 From ba8cf108735714d77430ff051cc2e414a2d82183 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 17:36:33 +0100 Subject: Rework libregrtest.save_env * Replace get/restore methods with a Resource class and Resource subclasses * Create ModuleAttr, ModuleAttrList and ModuleAttrDict helper classes * Use __subclasses__() to get resource classes instead of using an hardcoded list (2 shutil resources were missinged in the list!) * Don't define MultiprocessingProcessDangling resource if the multiprocessing module is missing * Nicer diff for dictionaries. Useful for the big os.environ dict * Reorder code to group resources --- Lib/test/libregrtest/save_env.py | 481 +++++++++++++++++++++++---------------- 1 file changed, 279 insertions(+), 202 deletions(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index 90900a9..c9a980c 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -17,225 +17,266 @@ except ImportError: multiprocessing = None -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. +class Resource: + name = None -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. + def __init__(self): + self.original = self.get() + self.current = None - with saved_test_environment(testname, verbose, quiet): - #stuff + def decode_value(self, value): + return value - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. + def display_diff(self): + original = self.decode_value(self.original) + current = self.decode_value(self.current) + print(f" Before: {original}", file=sys.stderr) + print(f" After: {current}", file=sys.stderr) - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - changed = False +class ModuleAttr(Resource): + def encode_value(self, value): + return value - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - self.pgo = pgo + def get_module(self): + module_name, attr = self.name.split('.', 1) + module = globals()[module_name] + return (module, attr) - # To add things to save and restore, add a name XXX to the resources list - # and add corresponding get_XXX/restore_XXX functions. get_XXX should - # return the value to be saved and compared against a second call to the - # get function when test execution completes. restore_XXX should accept - # the saved value and restore the resource using it. It will be called if - # and only if a change in the value is detected. - # - # Note: XXX will have any '.' replaced with '_' characters when determining - # the corresponding method names. - - resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', - 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', - 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', 'sys.gettrace', - 'sys.warnoptions', - # multiprocessing.process._cleanup() may release ref - # to a thread, so check processes first. - 'multiprocessing.process._dangling', 'threading._dangling', - 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', - 'files', 'locale', 'warnings.showwarning', - ) - - def get_sys_argv(self): - return id(sys.argv), sys.argv, sys.argv[:] - def restore_sys_argv(self, saved_argv): - sys.argv = saved_argv[1] - sys.argv[:] = saved_argv[2] - - def get_cwd(self): - return os.getcwd() - def restore_cwd(self, saved_cwd): - os.chdir(saved_cwd) - - def get_sys_stdout(self): - return sys.stdout - def restore_sys_stdout(self, saved_stdout): - sys.stdout = saved_stdout - - def get_sys_stderr(self): - return sys.stderr - def restore_sys_stderr(self, saved_stderr): - sys.stderr = saved_stderr - - def get_sys_stdin(self): - return sys.stdin - def restore_sys_stdin(self, saved_stdin): - sys.stdin = saved_stdin - - def get_os_environ(self): - return id(os.environ), os.environ, dict(os.environ) - def restore_os_environ(self, saved_environ): - os.environ = saved_environ[1] - os.environ.clear() - os.environ.update(saved_environ[2]) - - def get_sys_path(self): - return id(sys.path), sys.path, sys.path[:] - def restore_sys_path(self, saved_path): - sys.path = saved_path[1] - sys.path[:] = saved_path[2] - - def get_sys_path_hooks(self): - return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] - def restore_sys_path_hooks(self, saved_hooks): - sys.path_hooks = saved_hooks[1] - sys.path_hooks[:] = saved_hooks[2] - - def get_sys_gettrace(self): + def get(self): + module, attr = self.get_module() + value = getattr(module, attr) + return self.encode_value(value) + + def restore_attr(self, module, attr): + setattr(module, attr, self.original) + + def restore(self): + module, attr = self.get_module() + self.restore_attr(module, attr) + + +class ModuleAttrList(ModuleAttr): + def decode_value(self, value): + return value[2] + + def encode_value(self, value): + return id(value), value, value.copy() + + def restore_attr(self, module, attr): + value = self.original[1] + value[:] = self.original[2] + setattr(module, attr, value) + + +class ModuleAttrDict(ModuleAttr): + _MARKER = object() + + def encode_value(self, value): + return id(value), value, value.copy() + + def decode_value(self, value): + return value[2] + + def restore_attr(self, module, attr): + value = self.original[1] + value.clear() + value.update(self.original[2]) + setattr(module, attr, value) + + @classmethod + def _get_key(cls, data, key): + value = data.get(key, cls._MARKER) + if value is not cls._MARKER: + return repr(value) + else: + return '' + + def display_diff(self): + old_dict = self.original[2] + new_dict = self.current[2] + + keys = sorted(dict(old_dict).keys() | dict(new_dict).keys()) + for key in keys: + old_value = self._get_key(old_dict, key) + new_value = self._get_key(new_dict, key) + if old_value == new_value: + continue + + print(f" {self.name}[{key!r}]: {old_value} => {new_value}", + file=sys.stderr) + + +class SysArgv(ModuleAttrList): + name = 'sys.argv' + +class SysStdin(ModuleAttr): + name = 'sys.stdin' + +class SysStdout(ModuleAttr): + name = 'sys.stdout' + +class SysStderr(ModuleAttr): + name = 'sys.stderr' + +class SysPath(ModuleAttrList): + name = 'sys.path' + +class SysPathHooks(ModuleAttrList): + name = 'sys.path_hooks' + +class SysWarnOptions(ModuleAttrList): + name = 'sys.warnoptions' + +class SysGettrace(Resource): + name = 'sys.gettrace' + + def get(self): return sys.gettrace() - def restore_sys_gettrace(self, trace_fxn): - sys.settrace(trace_fxn) - def get___import__(self): - return builtins.__import__ - def restore___import__(self, import_): - builtins.__import__ = import_ + def restore(self): + sys.settrace(self.original) - def get_warnings_filters(self): - return id(warnings.filters), warnings.filters, warnings.filters[:] - def restore_warnings_filters(self, saved_filters): - warnings.filters = saved_filters[1] - warnings.filters[:] = saved_filters[2] - def get_asyncore_socket_map(self): +class OsEnviron(ModuleAttrDict): + name = 'os.environ' + +class ImportFunc(ModuleAttr): + name = 'builtins.__import__' + +class WarningsFilters(ModuleAttrList): + name = 'warnings.filters' + +class WarningsShowWarning(ModuleAttr): + name = 'warnings.showwarning' + + + + +class Cwd(Resource): + name = 'cwd' + + def get(self): + return os.getcwd() + + def restore(self): + os.chdir(self.original) + + +class AsyncoreSocketMap(Resource): + name = 'asyncore.socket_map' + + def get(self): asyncore = sys.modules.get('asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} - def restore_asyncore_socket_map(self, saved_map): + + def restore(self): asyncore = sys.modules.get('asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(saved_map) - - def get_shutil_archive_formats(self): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() - def restore_shutil_archive_formats(self, saved): - shutil._ARCHIVE_FORMATS = saved[0] - shutil._ARCHIVE_FORMATS.clear() - shutil._ARCHIVE_FORMATS.update(saved[1]) - - def get_shutil_unpack_formats(self): - return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() - def restore_shutil_unpack_formats(self, saved): - shutil._UNPACK_FORMATS = saved[0] - shutil._UNPACK_FORMATS.clear() - shutil._UNPACK_FORMATS.update(saved[1]) - - def get_logging__handlers(self): - # _handlers is a WeakValueDictionary - return id(logging._handlers), logging._handlers, logging._handlers.copy() - def restore_logging__handlers(self, saved_handlers): + asyncore.socket_map.update(self.original) + + +class ShutilUnpackFormats(ModuleAttrDict): + name = 'shutil._UNPACK_FORMATS' + +class ShutilArchiveFormats(ModuleAttrDict): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + + name = 'shutil._ARCHIVE_FORMATS' + + +class LoggingHandlers(ModuleAttrDict): + # _handlers is a WeakValueDictionary + name = 'logging._handlers' + + def restore(self): # Can't easily revert the logging state pass - def get_logging__handlerList(self): - # _handlerList is a list of weakrefs to handlers - return id(logging._handlerList), logging._handlerList, logging._handlerList[:] - def restore_logging__handlerList(self, saved_handlerList): +class LoggingHandlerList(ModuleAttrList): + # _handlerList is a list of weakrefs to handlers + name = 'logging._handlerList' + + def restore(self): # Can't easily revert the logging state pass - def get_sys_warnoptions(self): - return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] - def restore_sys_warnoptions(self, saved_options): - sys.warnoptions = saved_options[1] - sys.warnoptions[:] = saved_options[2] +class ThreadingDangling(Resource): # Controlling dangling references to Thread objects can make it easier # to track reference leaks. - def get_threading__dangling(self): + + name = 'threading._dangling' + + def get(self): if not threading: return None # This copies the weakrefs without making any strong reference return threading._dangling.copy() - def restore_threading__dangling(self, saved): + + def restore(self): if not threading: return threading._dangling.clear() - threading._dangling.update(saved) + threading._dangling.update(self.original) + + +if not multiprocessing: + class MultiprocessingProcessDangling(Resource): + # Same for Process objects + + name = 'multiprocessing.process._dangling' + + def get(self): + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + + def restore(self): + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(self.original) - # Same for Process objects - def get_multiprocessing_process__dangling(self): - if not multiprocessing: - return None - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - def restore_multiprocessing_process__dangling(self, saved): - if not multiprocessing: - return - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(saved) - def get_sysconfig__CONFIG_VARS(self): +class SysconfigInstallSchemes(ModuleAttrDict): + name = 'sysconfig._INSTALL_SCHEMES' + +class SysconfigConfigVars(ModuleAttrDict): + name = 'sysconfig._CONFIG_VARS' + + def get(self): # make sure the dict is initialized sysconfig.get_config_var('prefix') - return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, - dict(sysconfig._CONFIG_VARS)) - def restore_sysconfig__CONFIG_VARS(self, saved): - sysconfig._CONFIG_VARS = saved[1] - sysconfig._CONFIG_VARS.clear() - sysconfig._CONFIG_VARS.update(saved[2]) - - def get_sysconfig__INSTALL_SCHEMES(self): - return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, - sysconfig._INSTALL_SCHEMES.copy()) - def restore_sysconfig__INSTALL_SCHEMES(self, saved): - sysconfig._INSTALL_SCHEMES = saved[1] - sysconfig._INSTALL_SCHEMES.clear() - sysconfig._INSTALL_SCHEMES.update(saved[2]) - - def get_files(self): + return super().get() + + +class Files(Resource): + name = 'files' + + def get(self): return sorted(fn + ('/' if os.path.isdir(fn) else '') for fn in os.listdir()) - def restore_files(self, saved_value): + + def restore(self): fn = support.TESTFN - if fn not in saved_value and (fn + '/') not in saved_value: + if fn not in self.original and (fn + '/') not in self.original: if os.path.isfile(fn): support.unlink(fn) elif os.path.isdir(fn): support.rmtree(fn) + +class Locale(Resource): + name = 'locale' + _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] - def get_locale(self): + + def get(self): pairings = [] for lc in self._lc: try: @@ -243,43 +284,79 @@ class saved_test_environment: except (TypeError, ValueError): continue return pairings - def restore_locale(self, saved): - for lc, setting in saved: + + def restore(self): + for lc, setting in self.original: locale.setlocale(lc, setting) - def get_warnings_showwarning(self): - return warnings.showwarning - def restore_warnings_showwarning(self, fxn): - warnings.showwarning = fxn - def resource_info(self): - for name in self.resources: - method_suffix = name.replace('.', '_') - get_name = 'get_' + method_suffix - restore_name = 'restore_' + method_suffix - yield name, getattr(self, get_name), getattr(self, restore_name) +def _get_resources(parent_cls, resources): + for cls in parent_cls.__subclasses__(): + if cls.name is not None: + resources.append(cls) + _get_resources(cls, resources) + +def get_resources(resources=None): + resources = [] + _get_resources(Resource, resources) + return resources + + + +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. + +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. + + with saved_test_environment(testname, verbose, quiet): + #stuff + + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + + changed = False + + def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + self.pgo = pgo + self.resources = None def __enter__(self): - self.saved_values = dict((name, get()) for name, get, restore - in self.resource_info()) + self.resources = [resource_class() + for resource_class in get_resources()] return self def __exit__(self, exc_type, exc_val, exc_tb): - saved_values = self.saved_values - del self.saved_values - for name, get, restore in self.resource_info(): - current = get() - original = saved_values.pop(name) + # clear resources since they keep strong references to many objects + resources = self.resources + self.resources = None + + for resource in resources: + resource.current = resource.get() + # Check for changes to the resource's value - if current != original: - self.changed = True - restore(original) - if not self.quiet and not self.pgo: - print("Warning -- {} was modified by {}".format( - name, self.testname), - file=sys.stderr) - if self.verbose > 1: - print(" Before: {}\n After: {} ".format( - original, current), - file=sys.stderr) + if resource.current == resource.original: + continue + + self.changed = True + resource.restore() + if self.quiet or self.pgo: + continue + + print(f"Warning -- {resource.name} was modified by {self.testname}", + file=sys.stderr) + resource.display_diff() return False -- cgit v0.12 From d7ac00e6209ae29fbd4c9dd71885c32ad4c7340e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 19:13:06 +0100 Subject: Backed out changeset 245a16f33c4b Serhiy asked me to review it. --- Lib/test/libregrtest/save_env.py | 481 ++++++++++++++++----------------------- 1 file changed, 202 insertions(+), 279 deletions(-) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index c9a980c..90900a9 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -17,266 +17,225 @@ except ImportError: multiprocessing = None -class Resource: - name = None - - def __init__(self): - self.original = self.get() - self.current = None - - def decode_value(self, value): - return value - - def display_diff(self): - original = self.decode_value(self.original) - current = self.decode_value(self.current) - print(f" Before: {original}", file=sys.stderr) - print(f" After: {current}", file=sys.stderr) - - -class ModuleAttr(Resource): - def encode_value(self, value): - return value - - def get_module(self): - module_name, attr = self.name.split('.', 1) - module = globals()[module_name] - return (module, attr) - - def get(self): - module, attr = self.get_module() - value = getattr(module, attr) - return self.encode_value(value) - - def restore_attr(self, module, attr): - setattr(module, attr, self.original) - - def restore(self): - module, attr = self.get_module() - self.restore_attr(module, attr) - - -class ModuleAttrList(ModuleAttr): - def decode_value(self, value): - return value[2] - - def encode_value(self, value): - return id(value), value, value.copy() - - def restore_attr(self, module, attr): - value = self.original[1] - value[:] = self.original[2] - setattr(module, attr, value) - - -class ModuleAttrDict(ModuleAttr): - _MARKER = object() - - def encode_value(self, value): - return id(value), value, value.copy() - - def decode_value(self, value): - return value[2] - - def restore_attr(self, module, attr): - value = self.original[1] - value.clear() - value.update(self.original[2]) - setattr(module, attr, value) - - @classmethod - def _get_key(cls, data, key): - value = data.get(key, cls._MARKER) - if value is not cls._MARKER: - return repr(value) - else: - return '' - - def display_diff(self): - old_dict = self.original[2] - new_dict = self.current[2] - - keys = sorted(dict(old_dict).keys() | dict(new_dict).keys()) - for key in keys: - old_value = self._get_key(old_dict, key) - new_value = self._get_key(new_dict, key) - if old_value == new_value: - continue - - print(f" {self.name}[{key!r}]: {old_value} => {new_value}", - file=sys.stderr) - - -class SysArgv(ModuleAttrList): - name = 'sys.argv' - -class SysStdin(ModuleAttr): - name = 'sys.stdin' - -class SysStdout(ModuleAttr): - name = 'sys.stdout' - -class SysStderr(ModuleAttr): - name = 'sys.stderr' - -class SysPath(ModuleAttrList): - name = 'sys.path' - -class SysPathHooks(ModuleAttrList): - name = 'sys.path_hooks' - -class SysWarnOptions(ModuleAttrList): - name = 'sys.warnoptions' - -class SysGettrace(Resource): - name = 'sys.gettrace' - - def get(self): - return sys.gettrace() - - def restore(self): - sys.settrace(self.original) - - -class OsEnviron(ModuleAttrDict): - name = 'os.environ' - -class ImportFunc(ModuleAttr): - name = 'builtins.__import__' +# Unit tests are supposed to leave the execution environment unchanged +# once they complete. But sometimes tests have bugs, especially when +# tests fail, and the changes to environment go on to mess up other +# tests. This can cause issues with buildbot stability, since tests +# are run in random order and so problems may appear to come and go. +# There are a few things we can save and restore to mitigate this, and +# the following context manager handles this task. -class WarningsFilters(ModuleAttrList): - name = 'warnings.filters' +class saved_test_environment: + """Save bits of the test environment and restore them at block exit. -class WarningsShowWarning(ModuleAttr): - name = 'warnings.showwarning' + with saved_test_environment(testname, verbose, quiet): + #stuff + Unless quiet is True, a warning is printed to stderr if any of + the saved items was changed by the test. The attribute 'changed' + is initially False, but is set to True if a change is detected. + If verbose is more than 1, the before and after state of changed + items is also printed. + """ + changed = False -class Cwd(Resource): - name = 'cwd' + def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): + self.testname = testname + self.verbose = verbose + self.quiet = quiet + self.pgo = pgo - def get(self): + # To add things to save and restore, add a name XXX to the resources list + # and add corresponding get_XXX/restore_XXX functions. get_XXX should + # return the value to be saved and compared against a second call to the + # get function when test execution completes. restore_XXX should accept + # the saved value and restore the resource using it. It will be called if + # and only if a change in the value is detected. + # + # Note: XXX will have any '.' replaced with '_' characters when determining + # the corresponding method names. + + resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', + 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', + 'warnings.filters', 'asyncore.socket_map', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', + 'sys.warnoptions', + # multiprocessing.process._cleanup() may release ref + # to a thread, so check processes first. + 'multiprocessing.process._dangling', 'threading._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._INSTALL_SCHEMES', + 'files', 'locale', 'warnings.showwarning', + ) + + def get_sys_argv(self): + return id(sys.argv), sys.argv, sys.argv[:] + def restore_sys_argv(self, saved_argv): + sys.argv = saved_argv[1] + sys.argv[:] = saved_argv[2] + + def get_cwd(self): return os.getcwd() + def restore_cwd(self, saved_cwd): + os.chdir(saved_cwd) + + def get_sys_stdout(self): + return sys.stdout + def restore_sys_stdout(self, saved_stdout): + sys.stdout = saved_stdout + + def get_sys_stderr(self): + return sys.stderr + def restore_sys_stderr(self, saved_stderr): + sys.stderr = saved_stderr + + def get_sys_stdin(self): + return sys.stdin + def restore_sys_stdin(self, saved_stdin): + sys.stdin = saved_stdin + + def get_os_environ(self): + return id(os.environ), os.environ, dict(os.environ) + def restore_os_environ(self, saved_environ): + os.environ = saved_environ[1] + os.environ.clear() + os.environ.update(saved_environ[2]) + + def get_sys_path(self): + return id(sys.path), sys.path, sys.path[:] + def restore_sys_path(self, saved_path): + sys.path = saved_path[1] + sys.path[:] = saved_path[2] + + def get_sys_path_hooks(self): + return id(sys.path_hooks), sys.path_hooks, sys.path_hooks[:] + def restore_sys_path_hooks(self, saved_hooks): + sys.path_hooks = saved_hooks[1] + sys.path_hooks[:] = saved_hooks[2] + + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) - def restore(self): - os.chdir(self.original) - + def get___import__(self): + return builtins.__import__ + def restore___import__(self, import_): + builtins.__import__ = import_ -class AsyncoreSocketMap(Resource): - name = 'asyncore.socket_map' + def get_warnings_filters(self): + return id(warnings.filters), warnings.filters, warnings.filters[:] + def restore_warnings_filters(self, saved_filters): + warnings.filters = saved_filters[1] + warnings.filters[:] = saved_filters[2] - def get(self): + def get_asyncore_socket_map(self): asyncore = sys.modules.get('asyncore') # XXX Making a copy keeps objects alive until __exit__ gets called. return asyncore and asyncore.socket_map.copy() or {} - - def restore(self): + def restore_asyncore_socket_map(self, saved_map): asyncore = sys.modules.get('asyncore') if asyncore is not None: asyncore.close_all(ignore_all=True) - asyncore.socket_map.update(self.original) - - -class ShutilUnpackFormats(ModuleAttrDict): - name = 'shutil._UNPACK_FORMATS' - -class ShutilArchiveFormats(ModuleAttrDict): - # we could call get_archives_formats() but that only returns the - # registry keys; we want to check the values too (the functions that - # are registered) - - name = 'shutil._ARCHIVE_FORMATS' - - -class LoggingHandlers(ModuleAttrDict): - # _handlers is a WeakValueDictionary - name = 'logging._handlers' - - def restore(self): + asyncore.socket_map.update(saved_map) + + def get_shutil_archive_formats(self): + # we could call get_archives_formats() but that only returns the + # registry keys; we want to check the values too (the functions that + # are registered) + return shutil._ARCHIVE_FORMATS, shutil._ARCHIVE_FORMATS.copy() + def restore_shutil_archive_formats(self, saved): + shutil._ARCHIVE_FORMATS = saved[0] + shutil._ARCHIVE_FORMATS.clear() + shutil._ARCHIVE_FORMATS.update(saved[1]) + + def get_shutil_unpack_formats(self): + return shutil._UNPACK_FORMATS, shutil._UNPACK_FORMATS.copy() + def restore_shutil_unpack_formats(self, saved): + shutil._UNPACK_FORMATS = saved[0] + shutil._UNPACK_FORMATS.clear() + shutil._UNPACK_FORMATS.update(saved[1]) + + def get_logging__handlers(self): + # _handlers is a WeakValueDictionary + return id(logging._handlers), logging._handlers, logging._handlers.copy() + def restore_logging__handlers(self, saved_handlers): # Can't easily revert the logging state pass -class LoggingHandlerList(ModuleAttrList): - # _handlerList is a list of weakrefs to handlers - name = 'logging._handlerList' - - def restore(self): + def get_logging__handlerList(self): + # _handlerList is a list of weakrefs to handlers + return id(logging._handlerList), logging._handlerList, logging._handlerList[:] + def restore_logging__handlerList(self, saved_handlerList): # Can't easily revert the logging state pass + def get_sys_warnoptions(self): + return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] + def restore_sys_warnoptions(self, saved_options): + sys.warnoptions = saved_options[1] + sys.warnoptions[:] = saved_options[2] -class ThreadingDangling(Resource): # Controlling dangling references to Thread objects can make it easier # to track reference leaks. - - name = 'threading._dangling' - - def get(self): + def get_threading__dangling(self): if not threading: return None # This copies the weakrefs without making any strong reference return threading._dangling.copy() - - def restore(self): + def restore_threading__dangling(self, saved): if not threading: return threading._dangling.clear() - threading._dangling.update(self.original) + threading._dangling.update(saved) + # Same for Process objects + def get_multiprocessing_process__dangling(self): + if not multiprocessing: + return None + # Unjoined process objects can survive after process exits + multiprocessing.process._cleanup() + # This copies the weakrefs without making any strong reference + return multiprocessing.process._dangling.copy() + def restore_multiprocessing_process__dangling(self, saved): + if not multiprocessing: + return + multiprocessing.process._dangling.clear() + multiprocessing.process._dangling.update(saved) -if not multiprocessing: - class MultiprocessingProcessDangling(Resource): - # Same for Process objects - - name = 'multiprocessing.process._dangling' - - def get(self): - # Unjoined process objects can survive after process exits - multiprocessing.process._cleanup() - # This copies the weakrefs without making any strong reference - return multiprocessing.process._dangling.copy() - - def restore(self): - multiprocessing.process._dangling.clear() - multiprocessing.process._dangling.update(self.original) - - -class SysconfigInstallSchemes(ModuleAttrDict): - name = 'sysconfig._INSTALL_SCHEMES' - -class SysconfigConfigVars(ModuleAttrDict): - name = 'sysconfig._CONFIG_VARS' - - def get(self): + def get_sysconfig__CONFIG_VARS(self): # make sure the dict is initialized sysconfig.get_config_var('prefix') - return super().get() - - -class Files(Resource): - name = 'files' - - def get(self): + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__INSTALL_SCHEMES(self): + return (id(sysconfig._INSTALL_SCHEMES), sysconfig._INSTALL_SCHEMES, + sysconfig._INSTALL_SCHEMES.copy()) + def restore_sysconfig__INSTALL_SCHEMES(self, saved): + sysconfig._INSTALL_SCHEMES = saved[1] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(saved[2]) + + def get_files(self): return sorted(fn + ('/' if os.path.isdir(fn) else '') for fn in os.listdir()) - - def restore(self): + def restore_files(self, saved_value): fn = support.TESTFN - if fn not in self.original and (fn + '/') not in self.original: + if fn not in saved_value and (fn + '/') not in saved_value: if os.path.isfile(fn): support.unlink(fn) elif os.path.isdir(fn): support.rmtree(fn) - -class Locale(Resource): - name = 'locale' - _lc = [getattr(locale, lc) for lc in dir(locale) if lc.startswith('LC_')] - - def get(self): + def get_locale(self): pairings = [] for lc in self._lc: try: @@ -284,79 +243,43 @@ class Locale(Resource): except (TypeError, ValueError): continue return pairings - - def restore(self): - for lc, setting in self.original: + def restore_locale(self, saved): + for lc, setting in saved: locale.setlocale(lc, setting) + def get_warnings_showwarning(self): + return warnings.showwarning + def restore_warnings_showwarning(self, fxn): + warnings.showwarning = fxn -def _get_resources(parent_cls, resources): - for cls in parent_cls.__subclasses__(): - if cls.name is not None: - resources.append(cls) - _get_resources(cls, resources) - -def get_resources(resources=None): - resources = [] - _get_resources(Resource, resources) - return resources - - - -# Unit tests are supposed to leave the execution environment unchanged -# once they complete. But sometimes tests have bugs, especially when -# tests fail, and the changes to environment go on to mess up other -# tests. This can cause issues with buildbot stability, since tests -# are run in random order and so problems may appear to come and go. -# There are a few things we can save and restore to mitigate this, and -# the following context manager handles this task. - -class saved_test_environment: - """Save bits of the test environment and restore them at block exit. - - with saved_test_environment(testname, verbose, quiet): - #stuff - - Unless quiet is True, a warning is printed to stderr if any of - the saved items was changed by the test. The attribute 'changed' - is initially False, but is set to True if a change is detected. - - If verbose is more than 1, the before and after state of changed - items is also printed. - """ - - changed = False - - def __init__(self, testname, verbose=0, quiet=False, *, pgo=False): - self.testname = testname - self.verbose = verbose - self.quiet = quiet - self.pgo = pgo - self.resources = None + def resource_info(self): + for name in self.resources: + method_suffix = name.replace('.', '_') + get_name = 'get_' + method_suffix + restore_name = 'restore_' + method_suffix + yield name, getattr(self, get_name), getattr(self, restore_name) def __enter__(self): - self.resources = [resource_class() - for resource_class in get_resources()] + self.saved_values = dict((name, get()) for name, get, restore + in self.resource_info()) return self def __exit__(self, exc_type, exc_val, exc_tb): - # clear resources since they keep strong references to many objects - resources = self.resources - self.resources = None - - for resource in resources: - resource.current = resource.get() - + saved_values = self.saved_values + del self.saved_values + for name, get, restore in self.resource_info(): + current = get() + original = saved_values.pop(name) # Check for changes to the resource's value - if resource.current == resource.original: - continue - - self.changed = True - resource.restore() - if self.quiet or self.pgo: - continue - - print(f"Warning -- {resource.name} was modified by {self.testname}", - file=sys.stderr) - resource.display_diff() + if current != original: + self.changed = True + restore(original) + if not self.quiet and not self.pgo: + print("Warning -- {} was modified by {}".format( + name, self.testname), + file=sys.stderr) + if self.verbose > 1: + print(" Before: {}\n After: {} ".format( + original, current), + file=sys.stderr) return False -- cgit v0.12 From e984eb501b0b2a95d14b1d160c04eda5a88d19c5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 25 Mar 2016 22:51:17 +0100 Subject: Fix test_os.test_symlink(): remove create symlink --- Lib/test/test_os.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f7d64b7..1117875 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2554,6 +2554,8 @@ class Win32DeprecatedBytesAPI(unittest.TestCase): @support.skip_unless_symlink def test_symlink(self): + self.addCleanup(support.unlink, support.TESTFN) + filename = os.fsencode(support.TESTFN) with bytes_filename_warn(True): os.symlink(filename, filename) -- cgit v0.12 From ca9dbc7d880ac6e5e1df3219e34670b18998a346 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Mar 2016 01:04:37 +0100 Subject: makeopcodetargets.py: we need to import Lib/opcode.py Issue #20021: use importlib.machinery to import Lib/opcode.py and not an opcode module coming from somewhere else. makeopcodetargets.py is part of the Python build process and it is run by an external Python program, not the built Python program. Patch written by Serhiy Storchaka. --- Python/makeopcodetargets.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 7a57a06..023c9e6 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -3,14 +3,34 @@ (for compilers supporting computed gotos or "labels-as-values", such as gcc). """ -import opcode import os import sys +try: + from importlib.machinery import SourceFileLoader +except ImportError: + import imp + + def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib") + return imp.load_module(modname, *imp.find_module(modname, [modpath])) +else: + def find_module(modname): + """Finds and returns a module in the local dist/checkout. + """ + modpath = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "Lib", modname + ".py") + return SourceFileLoader(modname, modpath).load_module() + + def write_contents(f): """Write C code contents to the target file object. """ + opcode = find_module('opcode') targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): targets[op] = "TARGET_%s" % opname -- cgit v0.12 From 53b0a41d3103ab84f3ef9a1532bf95cff8819636 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Mar 2016 01:12:36 +0100 Subject: Issue #25911: more info on test_os failure --- Lib/test/test_os.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1117875..2fb7d1b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -892,7 +892,7 @@ class WalkTests(unittest.TestCase): # Walk bottom-up. all = list(self.walk(self.walk_path, topdown=False)) - self.assertEqual(len(all), 4) + self.assertEqual(len(all), 4, all) # We can't know which order SUB1 and SUB2 will appear in. # Not flipped: SUB11, SUB1, SUB2, TESTFN # flipped: SUB2, SUB11, SUB1, TESTFN -- cgit v0.12 From 942302371c1e46fdf486a5be0e036f7232a2fba1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Mar 2016 03:02:48 -0700 Subject: Minor code cleanup for PyArg_UnpackTuple. --- Python/getargs.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Python/getargs.c b/Python/getargs.c index 66a0c00..05ec27b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1771,16 +1771,9 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m PyObject **o; va_list vargs; -#ifdef HAVE_STDARG_PROTOTYPES - va_start(vargs, max); -#else - va_start(vargs); -#endif - assert(min >= 0); assert(min <= max); if (!PyTuple_Check(args)) { - va_end(vargs); PyErr_SetString(PyExc_SystemError, "PyArg_UnpackTuple() argument list is not a tuple"); return 0; @@ -1798,9 +1791,10 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m "unpacked tuple should have %s%zd elements," " but has %zd", (min == max ? "" : "at least "), min, l); - va_end(vargs); return 0; } + if (l == 0) + return 1; if (l > max) { if (name != NULL) PyErr_Format( @@ -1813,9 +1807,14 @@ PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t m "unpacked tuple should have %s%zd elements," " but has %zd", (min == max ? "" : "at most "), max, l); - va_end(vargs); return 0; } + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, max); +#else + va_start(vargs); +#endif for (i = 0; i < l; i++) { o = va_arg(vargs, PyObject **); *o = PyTuple_GET_ITEM(args, i); -- cgit v0.12 From 2c257ab0f85396664536605666af9a6f00c73a4f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 26 Mar 2016 04:10:11 -0700 Subject: Responsibility for argument checking belongs in set.__init__() rather than set.__new__(). See dict.__new__() and list.__new__() for comparison. Neither of those examine or touch args or kwds. That work is done in the __init__() methods. --- Objects/setobject.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 8e22b69..34235f8 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1131,9 +1131,6 @@ PySet_Fini(void) static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - if (kwds != NULL && type == &PySet_Type && !_PyArg_NoKeywords("set()", kwds)) - return NULL; - return make_new_set(type, NULL); } -- cgit v0.12 From 3840b2ac6728d1940a7efc2ecab2c7d9a100c080 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 27 Mar 2016 01:53:46 +0000 Subject: Issue #25940: Use internal local server more in test_ssl Move many tests from NetworkedTests and NetworkedBIOTests to a new Simple- BackgroundTests class, using the existing ThreadedEchoServer and SIGNED_ CERTFILE infrastructure. For tests that cause the server to crash by rejecting its certificate, separate them into independent test methods. Added custom root certificate to capath with the following commands: cp Lib/test/{pycacert.pem,capath/} # Edit copy to remove part before certificate c_rehash -v Lib/test/capath/ c_rehash -v -old Lib/test/capath/ # Note the generated file names cp Lib/test/capath/{pycacert.pem,b1930218.0} mv Lib/test/capath/{pycacert.pem,ceff1710.0} Change to pure PEM version of SIGNING_CA because PEM_cert_to_DER_cert() does not like the extra text at the start. Moved test_connect_ex_error() into BasicSocketTests and rewrote it to connect to a reserved localhost port. NetworkedTests.test_get_server_certificate_ipv6() split out because it needs to connect to an IPv6 DNS address. The only reference left to self-signed.pythontest.net is test_timeout_ connect_ex(), which needs a remote server to reliably time out the connection, but does not rely on the server running SSL. Made ThreadedEchoServer call unwrap() by default when it sees the client has shut the connection down, so that the client can cleanly call unwrap(). --- Lib/test/capath/0e4015b9.0 | 16 -- Lib/test/capath/b1930218.0 | 21 ++ Lib/test/capath/ce7b8643.0 | 16 -- Lib/test/capath/ceff1710.0 | 21 ++ Lib/test/test_ssl.py | 677 ++++++++++++++++++++++----------------------- Misc/NEWS | 2 +- 6 files changed, 378 insertions(+), 375 deletions(-) delete mode 100644 Lib/test/capath/0e4015b9.0 create mode 100644 Lib/test/capath/b1930218.0 delete mode 100644 Lib/test/capath/ce7b8643.0 create mode 100644 Lib/test/capath/ceff1710.0 diff --git a/Lib/test/capath/0e4015b9.0 b/Lib/test/capath/0e4015b9.0 deleted file mode 100644 index b6d259b..0000000 --- a/Lib/test/capath/0e4015b9.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/b1930218.0 b/Lib/test/capath/b1930218.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/Lib/test/capath/b1930218.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/capath/ce7b8643.0 b/Lib/test/capath/ce7b8643.0 deleted file mode 100644 index b6d259b..0000000 --- a/Lib/test/capath/ce7b8643.0 +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv -bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG -A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo -b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 -aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ -Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm -Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN -AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h -TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 -C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= ------END CERTIFICATE----- diff --git a/Lib/test/capath/ceff1710.0 b/Lib/test/capath/ceff1710.0 new file mode 100644 index 0000000..373349c --- /dev/null +++ b/Lib/test/capath/ceff1710.0 @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx +OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI +hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV +q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ +AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA +Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni +0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx +6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w +HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 +2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 +QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 +Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O +JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR +f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf +9mmvtk57HVjsO6lTo15YyJ4= +-----END CERTIFICATE----- diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e0c31a8..8b72ca9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -21,6 +21,13 @@ import functools ssl = support.import_module("ssl") +try: + import threading +except ImportError: + _have_threads = False +else: + _have_threads = True + PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = support.HOST @@ -53,10 +60,10 @@ CRLFILE = data_file("revocation.crl") # Two keys and certs signed by the same CA (for SNI tests) SIGNED_CERTFILE = data_file("keycert3.pem") SIGNED_CERTFILE2 = data_file("keycert4.pem") -SIGNING_CA = data_file("pycacert.pem") +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") REMOTE_HOST = "self-signed.pythontest.net" -REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") EMPTYCERT = data_file("nullcert.pem") BADCERT = data_file("badcert.pem") @@ -783,6 +790,22 @@ class BasicSocketTests(unittest.TestCase): self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = support.bind_port(server) # Reserve port but don't listen + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + class ContextTests(unittest.TestCase): @@ -1368,140 +1391,103 @@ class MemoryBIOTests(unittest.TestCase): self.assertRaises(TypeError, bio.write, 1) -class NetworkedTests(unittest.TestCase): +@unittest.skipUnless(_have_threads, "Needs threading module") +class SimpleBackgroundTests(unittest.TestCase): - def test_connect(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE) - try: - s.connect((REMOTE_HOST, 443)) - self.assertEqual({}, s.getpeercert()) - finally: - s.close() + """Tests that connect to a simple server running in the background""" - # this should fail because we have no verification certs - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() + def setUp(self): + server = ThreadedEchoServer(SIGNED_CERTFILE) + self.server_addr = (HOST, server.port) + server.__enter__() + self.addCleanup(server.__exit__, None, None, None) - # this should succeed because we specify the root cert - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - s.connect((REMOTE_HOST, 443)) - self.assertTrue(s.getpeercert()) - finally: - s.close() + def test_connect(self): + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + + # this should succeed because we specify the root cert + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_ex(self): # Issue #11326: check connect_ex() implementation - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) - self.assertTrue(s.getpeercert()) - finally: - s.close() + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) def test_non_blocking_connect_ex(self): # Issue #11326: non-blocking connect_ex() should allow handshake # to proceed after the socket gets ready. - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: try: - s.setblocking(False) - rc = s.connect_ex((REMOTE_HOST, 443)) - # EWOULDBLOCK under Windows, EINPROGRESS elsewhere - self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) - # Wait for connect to finish + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: select.select([], [s], [], 5.0) - # Non-blocking handshake - while True: - try: - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], [], 5.0) - except ssl.SSLWantWriteError: - select.select([], [s], [], 5.0) - # SSL established - self.assertTrue(s.getpeercert()) - finally: - s.close() - - def test_timeout_connect_ex(self): - # Issue #12065: on a timeout, connect_ex() should return the original - # errno (mimicking the behaviour of non-SSL sockets). - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT, - do_handshake_on_connect=False) - try: - s.settimeout(0.0000001) - rc = s.connect_ex((REMOTE_HOST, 443)) - if rc == 0: - self.skipTest("REMOTE_HOST responded too quickly") - self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) - finally: - s.close() - - def test_connect_ex_error(self): - with support.transient_internet(REMOTE_HOST): - s = ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_REQUIRED, - ca_certs=REMOTE_ROOT_CERT) - try: - rc = s.connect_ex((REMOTE_HOST, 444)) - # Issue #19919: Windows machines or VMs hosted on Windows - # machines sometimes return EWOULDBLOCK. - errors = ( - errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, - errno.EWOULDBLOCK, - ) - self.assertIn(rc, errors) - finally: - s.close() + # SSL established + self.assertTrue(s.getpeercert()) def test_connect_with_context(self): - with support.transient_internet(REMOTE_HOST): - # Same as test_connect, but with a separately created context - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - self.assertEqual({}, s.getpeercert()) - finally: - s.close() - # Same with a server hostname - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname=REMOTE_HOST) - s.connect((REMOTE_HOST, 443)) - s.close() - # This should fail because we have no verification certs - ctx.verify_mode = ssl.CERT_REQUIRED - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", - s.connect, (REMOTE_HOST, 443)) - s.close() - # This should succeed because we specify the root cert - ctx.load_verify_locations(REMOTE_ROOT_CERT) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + s = ctx.wrap_socket(socket.socket(socket.AF_INET)) + self.addCleanup(s.close) + self.assertRaisesRegex(ssl.SSLError, "certificate verify failed", + s.connect, self.server_addr) def test_connect_capath(self): # Verify server certificates using the `capath` argument @@ -1509,198 +1495,130 @@ class NetworkedTests(unittest.TestCase): # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must # contain both versions of each certificate (same content, different # filename) for this test to be portable across OpenSSL releases. - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - # Same with a bytes `capath` argument - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=BYTES_CAPATH) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) def test_connect_cadata(self): - with open(REMOTE_ROOT_CERT) as f: + with open(SIGNING_CA) as f: pem = f.read() der = ssl.PEM_cert_to_DER_cert(pem) - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=pem) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) - # same with DER - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(cadata=der) - with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: - s.connect((REMOTE_HOST, 443)) - cert = s.getpeercert() - self.assertTrue(cert) + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") def test_makefile_close(self): # Issue #5238: creating a file-like object with makefile() shouldn't # delay closing the underlying "real socket" (here tested with its # file descriptor, hence skipping the test under Windows). - with support.transient_internet(REMOTE_HOST): - ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) - ss.connect((REMOTE_HOST, 443)) - fd = ss.fileno() - f = ss.makefile() - f.close() - # The fd is still open + ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: os.read(fd, 0) - # Closing the SSL socket should close the fd too - ss.close() - gc.collect() - with self.assertRaises(OSError) as e: - os.read(fd, 0) - self.assertEqual(e.exception.errno, errno.EBADF) + self.assertEqual(e.exception.errno, errno.EBADF) def test_non_blocking_handshake(self): - with support.transient_internet(REMOTE_HOST): - s = socket.socket(socket.AF_INET) - s.connect((REMOTE_HOST, 443)) - s.setblocking(False) - s = ssl.wrap_socket(s, - cert_reqs=ssl.CERT_NONE, - do_handshake_on_connect=False) - count = 0 - while True: - try: - count += 1 - s.do_handshake() - break - except ssl.SSLWantReadError: - select.select([s], [], []) - except ssl.SSLWantWriteError: - select.select([], [s], []) - s.close() - if support.verbose: - sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = ssl.wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) def test_get_server_certificate(self): - def _test_get_server_certificate(host, port, cert=None): - with support.transient_internet(host): - pem = ssl.get_server_certificate((host, port)) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) - - try: - pem = ssl.get_server_certificate((host, port), - ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail - if support.verbose: - sys.stdout.write("%s\n" % x) - else: - self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) - pem = ssl.get_server_certificate((host, port), - ca_certs=cert) - if not pem: - self.fail("No server certificate on %s:%s!" % (host, port)) - if support.verbose: - sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - - _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) - if support.IPV6_ENABLED: - _test_get_server_certificate('ipv6.google.com', 443) + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) def test_ciphers(self): - remote = (REMOTE_HOST, 443) - with support.transient_internet(remote[0]): - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: - s.connect(remote) - with ssl.wrap_socket(socket.socket(socket.AF_INET), - cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: - s.connect(remote) - # Error checking can happen at instantiation or when connecting - with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): - with socket.socket(socket.AF_INET) as sock: - s = ssl.wrap_socket(sock, - cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") - s.connect(remote) - - def test_algorithms(self): - # Issue #8484: all algorithms should be available when verifying a - # certificate. - # SHA256 was added in OpenSSL 0.9.8 - if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): - self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) - # sha256.tbs-internet.com needs SNI to use the correct certificate - if not ssl.HAS_SNI: - self.skipTest("SNI needed for this test") - # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) - remote = ("sha256.tbs-internet.com", 443) - sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") - with support.transient_internet("sha256.tbs-internet.com"): - ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(sha256_cert) - s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname="sha256.tbs-internet.com") - try: - s.connect(remote) - if support.verbose: - sys.stdout.write("\nCipher with %r is %r\n" % - (remote, s.cipher())) - sys.stdout.write("Certificate is:\n%s\n" % - pprint.pformat(s.getpeercert())) - finally: - s.close() + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = ssl.wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) def test_get_ca_certs_capath(self): # capath certs are loaded on request - with support.transient_internet(REMOTE_HOST): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(capath=CAPATH) - self.assertEqual(ctx.get_ca_certs(), []) - s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect((REMOTE_HOST, 443)) - try: - cert = s.getpeercert() - self.assertTrue(cert) - finally: - s.close() - self.assertEqual(len(ctx.get_ca_certs()), 1) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) @needs_sni def test_context_setget(self): # Check that the context of a connected socket can be replaced. - with support.transient_internet(REMOTE_HOST): - ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - s = socket.socket(socket.AF_INET) - with ctx1.wrap_socket(s) as ss: - ss.connect((REMOTE_HOST, 443)) - self.assertIs(ss.context, ctx1) - self.assertIs(ss._sslobj.context, ctx1) - ss.context = ctx2 - self.assertIs(ss.context, ctx2) - self.assertIs(ss._sslobj.context, ctx2) - - -class NetworkedBIOTests(unittest.TestCase): + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s) as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): # A simple IO loop. Call func(*args) depending on the error we get @@ -1736,64 +1654,128 @@ class NetworkedBIOTests(unittest.TestCase): % (count, func.__name__)) return ret - def test_handshake(self): + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(SIGNING_CA) + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, 'localhost') + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + s = ssl.wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(support.IPV6_ENABLED, 'Needs IPv6') + def test_get_server_certificate_ipv6(self): + with support.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + def test_algorithms(self): + # Issue #8484: all algorithms should be available when verifying a + # certificate. + # SHA256 was added in OpenSSL 0.9.8 + if ssl.OPENSSL_VERSION_INFO < (0, 9, 8, 0, 15): + self.skipTest("SHA256 not available on %r" % ssl.OPENSSL_VERSION) + # sha256.tbs-internet.com needs SNI to use the correct certificate + if not ssl.HAS_SNI: + self.skipTest("SNI needed for this test") + # https://sha2.hboeck.de/ was used until 2011-01-08 (no route to host) + remote = ("sha256.tbs-internet.com", 443) + sha256_cert = os.path.join(os.path.dirname(__file__), "sha256.pem") + with support.transient_internet("sha256.tbs-internet.com"): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(REMOTE_ROOT_CERT) - ctx.check_hostname = True - sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) - self.assertIs(sslobj._sslobj.owner, sslobj) - self.assertIsNone(sslobj.cipher()) - self.assertIsNone(sslobj.shared_ciphers()) - self.assertRaises(ValueError, sslobj.getpeercert) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertIsNone(sslobj.get_channel_binding('tls-unique')) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - self.assertTrue(sslobj.cipher()) - self.assertIsNone(sslobj.shared_ciphers()) - self.assertTrue(sslobj.getpeercert()) - if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: - self.assertTrue(sslobj.get_channel_binding('tls-unique')) + ctx.load_verify_locations(sha256_cert) + s = ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="sha256.tbs-internet.com") try: - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - except ssl.SSLSyscallError: - # self-signed.pythontest.net probably shuts down the TCP - # connection without sending a secure shutdown message, and - # this is reported as SSL_ERROR_SYSCALL - pass - self.assertRaises(ssl.SSLError, sslobj.write, b'foo') - sock.close() + s.connect(remote) + if support.verbose: + sys.stdout.write("\nCipher with %r is %r\n" % + (remote, s.cipher())) + sys.stdout.write("Certificate is:\n%s\n" % + pprint.pformat(s.getpeercert())) + finally: + s.close() - def test_read_write_data(self): - with support.transient_internet(REMOTE_HOST): - sock = socket.socket(socket.AF_INET) - sock.connect((REMOTE_HOST, 443)) - incoming = ssl.MemoryBIO() - outgoing = ssl.MemoryBIO() - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ctx.verify_mode = ssl.CERT_NONE - sslobj = ctx.wrap_bio(incoming, outgoing, False) - self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) - req = b'GET / HTTP/1.0\r\n\r\n' - self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) - buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) - self.assertEqual(buf[:5], b'HTTP/') - self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) - sock.close() +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) -try: - import threading -except ImportError: - _have_threads = False -else: - _have_threads = True +if _have_threads: from test.ssl_servers import make_https_server class ThreadedEchoServer(threading.Thread): @@ -1881,6 +1863,15 @@ else: if not stripped: # eof, so quit this handler self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None self.close() elif stripped == b'over': if support.verbose and self.server.connectionchatty: @@ -3342,18 +3333,20 @@ def test_main(verbose=False): pass for filename in [ - CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, + CERTFILE, BYTES_CERTFILE, ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, BADCERT, BADKEY, EMPTYCERT]: if not os.path.exists(filename): raise support.TestFailed("Can't read certificate file %r" % filename) - tests = [ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] + tests = [ + ContextTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests, + SimpleBackgroundTests, + ] if support.is_resource_enabled('network'): tests.append(NetworkedTests) - tests.append(NetworkedBIOTests) if _have_threads: thread_info = support.threading_setup() diff --git a/Misc/NEWS b/Misc/NEWS index ca36123..83dee73 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -873,7 +873,7 @@ Tests - Issue #26325: Added test.support.check_no_resource_warning() to check that no ResourceWarning is emitted. -- Issue #25940: Changed test_ssl to use self-signed.pythontest.net. This +- Issue #25940: Changed test_ssl to use its internal local server more. This avoids relying on svn.python.org, which recently changed root certificate. - Issue #25616: Tests for OrderedDict are extracted from test_collections -- cgit v0.12 From 3625af5f2129f8e8735c54bd01d9f138dd5c0e83 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 27 Mar 2016 01:15:07 -0700 Subject: Moved misplaced functions to the section for C API functions. --- Objects/setobject.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Objects/setobject.c b/Objects/setobject.c index 34235f8..176570e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1116,18 +1116,6 @@ frozenset_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return emptyfrozenset; } -int -PySet_ClearFreeList(void) -{ - return 0; -} - -void -PySet_Fini(void) -{ - Py_CLEAR(emptyfrozenset); -} - static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -2339,6 +2327,18 @@ PySet_Add(PyObject *anyset, PyObject *key) } int +PySet_ClearFreeList(void) +{ + return 0; +} + +void +PySet_Fini(void) +{ + Py_CLEAR(emptyfrozenset); +} + +int _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, Py_hash_t *hash) { setentry *entry; -- cgit v0.12 From 622583e9bf19b79b621eba8c5798acdf175a17d2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 27 Mar 2016 18:28:15 +0200 Subject: regrtest: round final timing towards +inf --- Lib/test/libregrtest/main.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index e1367da..447d99f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -1,5 +1,6 @@ import datetime import faulthandler +import math import os import platform import random @@ -106,9 +107,13 @@ class Regrtest: self.skipped.append(test) self.resource_denieds.append(test) - def time_delta(self): + def time_delta(self, ceil=False): seconds = time.monotonic() - self.start_time - return datetime.timedelta(seconds=int(seconds)) + if ceil: + seconds = math.ceil(seconds) + else: + seconds = int(seconds) + return datetime.timedelta(seconds=seconds) def display_progress(self, test_index, test): if self.ns.quiet: @@ -409,7 +414,7 @@ class Regrtest: r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) - print("Total duration: %s" % self.time_delta()) + print("Total duration: %s" % self.time_delta(ceil=True)) if self.ns.runleaks: os.system("leaks %d" % os.getpid()) -- cgit v0.12 From 2a65ecb780e2a5cd47bc4b9af947b1127e972a11 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Mon, 28 Mar 2016 00:45:28 +0300 Subject: Issue #26130: Remove redundant variable 's' from Parser/parser.c Patch by Oren Milman. --- Parser/parser.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index 56ec514..41072c4 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -140,21 +140,20 @@ classify(parser_state *ps, int type, const char *str) int n = g->g_ll.ll_nlabels; if (type == NAME) { - const char *s = str; label *l = g->g_ll.ll_label; int i; for (i = n; i > 0; i--, l++) { if (l->lb_type != NAME || l->lb_str == NULL || - l->lb_str[0] != s[0] || - strcmp(l->lb_str, s) != 0) + l->lb_str[0] != str[0] || + strcmp(l->lb_str, str) != 0) continue; #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD #if 0 /* Leaving this in as an example */ if (!(ps->p_flags & CO_FUTURE_WITH_STATEMENT)) { - if (s[0] == 'w' && strcmp(s, "with") == 0) + if (str[0] == 'w' && strcmp(str, "with") == 0) break; /* not a keyword yet */ - else if (s[0] == 'a' && strcmp(s, "as") == 0) + else if (str[0] == 'a' && strcmp(str, "as") == 0) break; /* not a keyword yet */ } #endif -- cgit v0.12 From 80ec58c49730cb5498a279ee796ff037e8bbb3a7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Mar 2016 09:50:18 +0200 Subject: fix typo in comment Thanks Arfrever for the report :) --- Lib/test/libregrtest/runtest_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtest_mp.py b/Lib/test/libregrtest/runtest_mp.py index aa97f05..33f632d 100644 --- a/Lib/test/libregrtest/runtest_mp.py +++ b/Lib/test/libregrtest/runtest_mp.py @@ -21,7 +21,7 @@ from test.libregrtest.setup import setup_tests # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds -# If interrupted, display the wait process every N seconds +# If interrupted, display the wait progress every N seconds WAIT_PROGRESS = 2.0 # seconds -- cgit v0.12 From 73030df77f05426872cee7512dc4632ccb35bcc4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 29 Mar 2016 11:25:00 +0200 Subject: Fix os._DummyDirEntry.is_symlink() Issue #25911: Fix os._DummyDirEntry.is_symlink(), don't follow symbolic links: use os.stat(path, follow_symlinks=False). --- Lib/os.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/os.py b/Lib/os.py index c3ce05d..90646a0 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -452,22 +452,33 @@ class _DummyDirEntry: # Mimick FindFirstFile/FindNextFile: we should get file attributes # while iterating on a directory self._stat = None + self._lstat = None try: - self.stat() + self.stat(follow_symlinks=False) except OSError: pass - def stat(self): - if self._stat is None: - self._stat = stat(self.path) - return self._stat + def stat(self, *, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + self._stat = stat(self.path) + return self._stat + else: + if self._lstat is None: + self._lstat = stat(self.path, follow_symlinks=False) + return self._lstat def is_dir(self): + if self._lstat is not None and not self.is_symlink(): + # use the cache lstat + stat = self.stat(follow_symlinks=False) + return st.S_ISDIR(stat.st_mode) + stat = self.stat() return st.S_ISDIR(stat.st_mode) def is_symlink(self): - stat = self.stat() + stat = self.stat(follow_symlinks=False) return st.S_ISLNK(stat.st_mode) class _dummy_scandir: -- cgit v0.12 From b1511f789ea0565296cf9c64f30d2ab4e79b8615 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 01:29:05 +0200 Subject: doctest now supports packages Issue #26641: doctest.DocFileTest and doctest.testfile() now support packages (module splitted into multiple directories) for the package parameter. --- Lib/doctest.py | 17 +++++++++++++---- Misc/NEWS | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index 38fdd80..5630220 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -381,12 +381,15 @@ class _OutputRedirectingPdb(pdb.Pdb): sys.stdout = save_stdout # [XX] Normalize with respect to os.path.pardir? -def _module_relative_path(module, path): +def _module_relative_path(module, test_path): if not inspect.ismodule(module): raise TypeError('Expected a module: %r' % module) - if path.startswith('/'): + if test_path.startswith('/'): raise ValueError('Module-relative files may not have absolute paths') + # Normalize the path. On Windows, replace "/" with "\". + test_path = os.path.join(*(test_path.split('/'))) + # Find the base directory for the path. if hasattr(module, '__file__'): # A normal module/package @@ -398,13 +401,19 @@ def _module_relative_path(module, path): else: basedir = os.curdir else: + if hasattr(module, '__path__'): + for directory in module.__path__: + fullpath = os.path.join(directory, test_path) + if os.path.exists(fullpath): + return fullpath + # A module w/o __file__ (this includes builtins) raise ValueError("Can't resolve paths relative to the module " "%r (it has no __file__)" % module.__name__) - # Combine the base directory and the path. - return os.path.join(basedir, *(path.split('/'))) + # Combine the base directory and the test path. + return os.path.join(basedir, test_path) ###################################################################### ## 2. Example & DocTest diff --git a/Misc/NEWS b/Misc/NEWS index 3923af1..73c355d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,10 @@ Core and Builtins Library ------- +- Issue #26641: doctest.DocFileTest and doctest.testfile() now support + packages (module splitted into multiple directories) for the package + parameter. + - Issue #25195: Fix a regression in mock.MagicMock. _Call is a subclass of tuple (changeset 3603bae63c13 only works for classes) so we need to implement __ne__ ourselves. Patch by Andrew Plummer. -- cgit v0.12 From 9759dd334325f341318ff5f2ef25409a5e44dc98 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 02:32:52 +0200 Subject: Issue #26295: When using "python3 -m test --testdir=TESTDIR", regrtest doesn't add "test." prefix to test module names. regrtest also prepends testdir to sys.path. --- Lib/test/libregrtest/runtest.py | 8 ++++---- Lib/test/libregrtest/setup.py | 5 +++++ Misc/NEWS | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index daff476..601f2b2 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -116,7 +116,7 @@ def runtest(ns, test): try: sys.stdout = stream sys.stderr = stream - result = runtest_inner(test, verbose, quiet, huntrleaks, + result = runtest_inner(ns, test, verbose, quiet, huntrleaks, display_failure=False, pgo=pgo) if result[0] == FAILED: output = stream.getvalue() @@ -127,7 +127,7 @@ def runtest(ns, test): sys.stderr = orig_stderr else: support.verbose = verbose # Tell tests to be moderately quiet - result = runtest_inner(test, verbose, quiet, huntrleaks, + result = runtest_inner(ns, test, verbose, quiet, huntrleaks, display_failure=not verbose, pgo=pgo) return result finally: @@ -137,14 +137,14 @@ def runtest(ns, test): runtest.stringio = None -def runtest_inner(test, verbose, quiet, +def runtest_inner(ns, test, verbose, quiet, huntrleaks=False, display_failure=True, *, pgo=False): support.unload(test) test_time = 0.0 refleak = False # True if the test leaked references. try: - if test.startswith('test.'): + if test.startswith('test.') or ns.testdir: abstest = test else: # Always import it from the test package diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index de52bb5..1d24531 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -29,6 +29,11 @@ def setup_tests(ns): replace_stdout() support.record_original_stdout(sys.stdout) + if ns.testdir: + # Prepend test directory to sys.path, so runtest() will be able + # to locate tests + sys.path.insert(0, os.path.abspath(ns.testdir)) + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). diff --git a/Misc/NEWS b/Misc/NEWS index 73c355d..34ef0cd 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -879,6 +879,9 @@ Documentation Tests ----- +- Issue #26295: When using "python3 -m test --testdir=TESTDIR", regrtest + doesn't add "test." prefix to test module names. + - Issue #26523: The multiprocessing thread pool (multiprocessing.dummy.Pool) was untested. -- cgit v0.12 From d6e2502624c3f19593657086e1d673d2dd699b21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 02:33:52 +0200 Subject: Issue #26295: test_regrtest now uses a temporary directory test_forever() stores its state into the builtins module since the test module is reloaded at each run. Remove also warning to detect leaked tests of a previous run. --- Lib/test/test_regrtest.py | 55 ++++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index df8447c..013a1b4 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -15,6 +15,7 @@ import re import subprocess import sys import sysconfig +import tempfile import textwrap import unittest from test import libregrtest @@ -309,15 +310,8 @@ class BaseTestCase(unittest.TestCase): def setUp(self): self.testdir = os.path.realpath(os.path.dirname(__file__)) - # When test_regrtest is interrupted by CTRL+c, it can leave - # temporary test files - remove = [entry.path - for entry in os.scandir(self.testdir) - if (entry.name.startswith(self.TESTNAME_PREFIX) - and entry.name.endswith(".py"))] - for path in remove: - print("WARNING: test_regrtest: remove %s" % path) - support.unlink(path) + self.tmptestdir = tempfile.mkdtemp() + self.addCleanup(support.rmtree, self.tmptestdir) def create_test(self, name=None, code=''): if not name: @@ -326,8 +320,8 @@ class BaseTestCase(unittest.TestCase): # test_regrtest cannot be run twice in parallel because # of setUp() and create_test() - name = self.TESTNAME_PREFIX + "%s_%s" % (os.getpid(), name) - path = os.path.join(self.testdir, name + '.py') + name = self.TESTNAME_PREFIX + name + path = os.path.join(self.tmptestdir, name + '.py') self.addCleanup(support.unlink, path) # Use 'x' mode to ensure that we do not override existing tests @@ -462,7 +456,8 @@ class ProgramsTestCase(BaseTestCase): self.tests = [self.create_test() for index in range(self.NTEST)] self.python_args = ['-Wd', '-E', '-bb'] - self.regrtest_args = ['-uall', '-rwW'] + self.regrtest_args = ['-uall', '-rwW', + '--testdir=%s' % self.tmptestdir] if hasattr(faulthandler, 'dump_traceback_later'): self.regrtest_args.extend(('--timeout', '3600', '-j4')) if sys.platform == 'win32': @@ -519,7 +514,8 @@ class ProgramsTestCase(BaseTestCase): def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - self.run_tests([script, *self.tests]) + args = [script, '--testdir=%s' % self.tmptestdir, *self.tests] + self.run_tests(args) def run_batch(self, *args): proc = self.run_command(args) @@ -555,8 +551,9 @@ class ArgsTestCase(BaseTestCase): Test arguments of the Python test suite. """ - def run_tests(self, *args, **kw): - return self.run_python(['-m', 'test', *args], **kw) + def run_tests(self, *testargs, **kw): + cmdargs = ['-m', 'test', '--testdir=%s' % self.tmptestdir, *testargs] + return self.run_python(cmdargs, **kw) def test_failing_test(self): # test a failing test @@ -567,8 +564,8 @@ class ArgsTestCase(BaseTestCase): def test_failing(self): self.fail("bug") """) - test_ok = self.create_test() - test_failing = self.create_test(code=code) + test_ok = self.create_test('ok') + test_failing = self.create_test('failing', code=code) tests = [test_ok, test_failing] output = self.run_tests(*tests, exitcode=1) @@ -661,7 +658,7 @@ class ArgsTestCase(BaseTestCase): def test_interrupted(self): code = TEST_INTERRUPTED - test = self.create_test("sigint", code=code) + test = self.create_test('sigint', code=code) output = self.run_tests(test, exitcode=1) self.check_executed_tests(output, test, omitted=test) @@ -693,7 +690,7 @@ class ArgsTestCase(BaseTestCase): def test_coverage(self): # test --coverage - test = self.create_test() + test = self.create_test('coverage') output = self.run_tests("--coverage", test) self.check_executed_tests(output, [test]) regex = ('lines +cov% +module +\(path\)\n' @@ -702,24 +699,28 @@ class ArgsTestCase(BaseTestCase): def test_wait(self): # test --wait - test = self.create_test() + test = self.create_test('wait') output = self.run_tests("--wait", test, input='key') self.check_line(output, 'Press any key to continue') def test_forever(self): # test --forever code = textwrap.dedent(""" + import builtins import unittest class ForeverTester(unittest.TestCase): - RUN = 1 - def test_run(self): - ForeverTester.RUN += 1 - if ForeverTester.RUN > 3: - self.fail("fail at the 3rd runs") + # Store the state in the builtins module, because the test + # module is reload at each run + if 'RUN' in builtins.__dict__: + builtins.__dict__['RUN'] += 1 + if builtins.__dict__['RUN'] >= 3: + self.fail("fail at the 3rd runs") + else: + builtins.__dict__['RUN'] = 1 """) - test = self.create_test(code=code) + test = self.create_test('forever', code=code) output = self.run_tests('--forever', test, exitcode=1) self.check_executed_tests(output, [test]*3, failed=test) @@ -747,7 +748,7 @@ class ArgsTestCase(BaseTestCase): fd = os.open(__file__, os.O_RDONLY) # bug: never cloes the file descriptor """) - test = self.create_test(code=code) + test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(support.unlink, filename) -- cgit v0.12 From 732599f793281d15a1d28df0ebe63fe1fba52780 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 08:38:05 +0200 Subject: Issue #26295: Fix test_regrtest.test_tools_buildbot_test() Pass also --testdir option. --- Lib/test/test_regrtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 013a1b4..b4083b8 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -527,7 +527,7 @@ class ProgramsTestCase(BaseTestCase): def test_tools_buildbot_test(self): # Tools\buildbot\test.bat script = os.path.join(ROOT_DIR, 'Tools', 'buildbot', 'test.bat') - test_args = [] + test_args = ['--testdir=%s' % self.tmptestdir] if platform.architecture()[0] == '64bit': test_args.append('-x64') # 64-bit build if not Py_DEBUG: -- cgit v0.12 From 8a34d416ba9e107e07fca7d4a1692b4b6a54a3ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 30 Mar 2016 08:51:15 +0200 Subject: Issue #26295: Enhanc test_regrtest.test_tools_script_run_tests() Pass all regrtest options, not only --testdir. --- Lib/test/test_regrtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index b4083b8..213853f 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -514,7 +514,7 @@ class ProgramsTestCase(BaseTestCase): def test_tools_script_run_tests(self): # Tools/scripts/run_tests.py script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py') - args = [script, '--testdir=%s' % self.tmptestdir, *self.tests] + args = [script, *self.regrtest_args, *self.tests] self.run_tests(args) def run_batch(self, *args): -- cgit v0.12 From ab0d198c7a6b2133ba5f782d7b951fef00ae615c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 30 Mar 2016 21:11:16 +0300 Subject: Issue #26492: Exhausted iterator of array.array now conforms with the behavior of iterators of other mutable sequences: it lefts exhausted even if iterated array is extended. --- Lib/test/test_array.py | 21 +++++++++++++++++++-- Misc/NEWS | 4 ++++ Modules/arraymodule.c | 22 ++++++++++++++++++---- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 482526e..b4f2bf8 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -318,8 +318,19 @@ class BaseTest: d = pickle.dumps((itorig, orig), proto) it, a = pickle.loads(d) a.fromlist(data2) - self.assertEqual(type(it), type(itorig)) - self.assertEqual(list(it), data2) + self.assertEqual(list(it), []) + + def test_exhausted_iterator(self): + a = array.array(self.typecode, self.example) + self.assertEqual(list(a), list(self.example)) + exhit = iter(a) + empit = iter(a) + for x in exhit: # exhaust the iterator + next(empit) # not exhausted + a.append(self.outside) + self.assertEqual(list(exhit), []) + self.assertEqual(list(empit), [self.outside]) + self.assertEqual(list(a), list(self.example) + [self.outside]) def test_insert(self): a = array.array(self.typecode, self.example) @@ -1070,6 +1081,12 @@ class BaseTest: a = array.array('B', b"") self.assertRaises(BufferError, getbuffer_with_null_view, a) + def test_free_after_iterating(self): + support.check_free_after_iterating(self, iter, array.array, + (self.typecode,)) + support.check_free_after_iterating(self, reversed, array.array, + (self.typecode,)) + class StringTest(BaseTest): def test_setitem(self): diff --git a/Misc/NEWS b/Misc/NEWS index 6fb0fc4..6b6d11c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,10 @@ Core and Builtins Library ------- +- Issue #26492: Exhausted iterator of array.array now conforms with the behavior + of iterators of other mutable sequences: it lefts exhausted even if iterated + array is extended. + - Issue #26641: doctest.DocFileTest and doctest.testfile() now support packages (module splitted into multiple directories) for the package parameter. diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 1b0a282..323e0c1 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -2875,9 +2875,20 @@ array_iter(arrayobject *ao) static PyObject * arrayiter_next(arrayiterobject *it) { + arrayobject *ao; + + assert(it != NULL); assert(PyArrayIter_Check(it)); - if (it->index < Py_SIZE(it->ao)) - return (*it->getitem)(it->ao, it->index++); + ao = it->ao; + if (ao == NULL) { + return NULL; + } + assert(array_Check(ao)); + if (it->index < Py_SIZE(ao)) { + return (*it->getitem)(ao, it->index++); + } + it->ao = NULL; + Py_DECREF(ao); return NULL; } @@ -2906,8 +2917,11 @@ static PyObject * array_arrayiterator___reduce___impl(arrayiterobject *self) /*[clinic end generated code: output=7898a52e8e66e016 input=a062ea1e9951417a]*/ { - return Py_BuildValue("N(O)n", _PyObject_GetBuiltin("iter"), - self->ao, self->index); + PyObject *func = _PyObject_GetBuiltin("iter"); + if (self->ao == NULL) { + return Py_BuildValue("N(())", func); + } + return Py_BuildValue("N(O)n", func, self->ao, self->index); } /*[clinic input] -- cgit v0.12 From c0aab1da3bb0c5f8b6b7fc20278f05ebcf2f0994 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Thu, 31 Mar 2016 10:31:30 +0000 Subject: Issue #22854: Skip pipe seekable() tests on Windows --- Lib/test/test_io.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 1944a04..9c410e7 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -425,7 +425,12 @@ class IOTest(unittest.TestCase): writable = "w" in abilities self.assertEqual(obj.writable(), writable) seekable = "s" in abilities - self.assertEqual(obj.seekable(), seekable) + + # Detection of pipes being non-seekable does not seem to work + # on Windows + if not sys.platform.startswith("win") or test not in ( + pipe_reader, pipe_writer): + self.assertEqual(obj.seekable(), seekable) if isinstance(obj, self.TextIOBase): data = "3" -- cgit v0.12 From 49f324f1d41668dbc478fa15f85921d0b05ec09b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 31 Mar 2016 23:30:53 +0200 Subject: Python 8: no pep8, no chocolate! --- Include/patchlevel.h | 6 +- Lib/pep8.py | 2151 ++++++++++++++++++++++++++++++++++++++++++++++++++ Lib/site.py | 56 ++ 3 files changed, 2210 insertions(+), 3 deletions(-) create mode 100644 Lib/pep8.py diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 246eba8..6237ef7 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -16,14 +16,14 @@ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 6 +#define PY_MAJOR_VERSION 8 +#define PY_MINOR_VERSION 0 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.6.0a0" +#define PY_VERSION "8.0.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pep8.py b/Lib/pep8.py new file mode 100644 index 0000000..3c950d4 --- /dev/null +++ b/Lib/pep8.py @@ -0,0 +1,2151 @@ +#!/usr/bin/env python +# pep8.py - Check Python source code formatting, according to PEP 8 +# Copyright (C) 2006-2009 Johann C. Rocholl +# Copyright (C) 2009-2014 Florent Xicluna +# Copyright (C) 2014-2016 Ian Lee +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +r""" +Check Python source code formatting, according to PEP 8. + +For usage and a list of options, try this: +$ python pep8.py -h + +This program and its regression test suite live here: +https://github.com/pycqa/pep8 + +Groups of errors and warnings: +E errors +W warnings +100 indentation +200 whitespace +300 blank lines +400 imports +500 line length +600 deprecation +700 statements +900 syntax error +""" +from __future__ import with_statement + +import os +import sys +import re +import time +import inspect +import keyword +import tokenize +from optparse import OptionParser +from fnmatch import fnmatch +try: + from configparser import RawConfigParser + from io import TextIOWrapper +except ImportError: + from ConfigParser import RawConfigParser + +__version__ = '1.7.0' + +DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' +DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' +try: + if sys.platform == 'win32': + USER_CONFIG = os.path.expanduser(r'~\.pep8') + else: + USER_CONFIG = os.path.join( + os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), + 'pep8' + ) +except ImportError: + USER_CONFIG = None + +PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') +TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') +MAX_LINE_LENGTH = 79 +REPORT_FORMAT = { + 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', + 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', +} + +PyCF_ONLY_AST = 1024 +SINGLETONS = frozenset(['False', 'None', 'True']) +KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS +UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) +ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) +WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) +WS_NEEDED_OPERATORS = frozenset([ + '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', + '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) +WHITESPACE = frozenset(' \t') +NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) +SKIP_TOKENS = NEWLINE.union([tokenize.INDENT, tokenize.DEDENT]) +# ERRORTOKEN is triggered by backticks in Python 3 +SKIP_COMMENTS = SKIP_TOKENS.union([tokenize.COMMENT, tokenize.ERRORTOKEN]) +BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] + +INDENT_REGEX = re.compile(r'([ \t]*)') +RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,') +RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,.*,\s*\w+\s*$') +ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') +DOCSTRING_REGEX = re.compile(r'u?r?["\']') +EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') +WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') +COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' + r'\s*(?(1)|(None|False|True))\b') +COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') +COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' + r'|\s*\(\s*([^)]*[^ )])\s*\))') +KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) +OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') +LAMBDA_REGEX = re.compile(r'\blambda\b') +HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') + +# Work around Python < 2.6 behaviour, which does not generate NL after +# a comment which is on a line by itself. +COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' + + +############################################################################## +# Plugins (check functions) for physical lines +############################################################################## + + +def tabs_or_spaces(physical_line, indent_char): + r"""Never mix tabs and spaces. + + The most popular way of indenting Python is with spaces only. The + second-most popular way is with tabs only. Code indented with a mixture + of tabs and spaces should be converted to using spaces exclusively. When + invoking the Python command line interpreter with the -t option, it issues + warnings about code that illegally mixes tabs and spaces. When using -tt + these warnings become errors. These options are highly recommended! + + Okay: if a == 0:\n a = 1\n b = 1 + E101: if a == 0:\n a = 1\n\tb = 1 + """ + indent = INDENT_REGEX.match(physical_line).group(1) + for offset, char in enumerate(indent): + if char != indent_char: + return offset, "E101 indentation contains mixed spaces and tabs" + + +def tabs_obsolete(physical_line): + r"""For new projects, spaces-only are strongly recommended over tabs. + + Okay: if True:\n return + W191: if True:\n\treturn + """ + indent = INDENT_REGEX.match(physical_line).group(1) + if '\t' in indent: + return indent.index('\t'), "W191 indentation contains tabs" + + +def trailing_whitespace(physical_line): + r"""Trailing whitespace is superfluous. + + The warning returned varies on whether the line itself is blank, for easier + filtering for those who want to indent their blank lines. + + Okay: spam(1)\n# + W291: spam(1) \n# + W293: class Foo(object):\n \n bang = 12 + """ + physical_line = physical_line.rstrip('\n') # chr(10), newline + physical_line = physical_line.rstrip('\r') # chr(13), carriage return + physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L + stripped = physical_line.rstrip(' \t\v') + if physical_line != stripped: + if stripped: + return len(stripped), "W291 trailing whitespace" + else: + return 0, "W293 blank line contains whitespace" + + +def trailing_blank_lines(physical_line, lines, line_number, total_lines): + r"""Trailing blank lines are superfluous. + + Okay: spam(1) + W391: spam(1)\n + + However the last line should end with a new line (warning W292). + """ + if line_number == total_lines: + stripped_last_line = physical_line.rstrip() + if not stripped_last_line: + return 0, "W391 blank line at end of file" + if stripped_last_line == physical_line: + return len(physical_line), "W292 no newline at end of file" + + +def maximum_line_length(physical_line, max_line_length, multiline): + r"""Limit all lines to a maximum of 79 characters. + + There are still many devices around that are limited to 80 character + lines; plus, limiting windows to 80 characters makes it possible to have + several windows side-by-side. The default wrapping on such devices looks + ugly. Therefore, please limit all lines to a maximum of 79 characters. + For flowing long blocks of text (docstrings or comments), limiting the + length to 72 characters is recommended. + + Reports error E501. + """ + line = physical_line.rstrip() + length = len(line) + if length > max_line_length and not noqa(line): + # Special case for long URLs in multi-line docstrings or comments, + # but still report the error when the 72 first chars are whitespaces. + chunks = line.split() + if ((len(chunks) == 1 and multiline) or + (len(chunks) == 2 and chunks[0] == '#')) and \ + len(line) - len(chunks[-1]) < max_line_length - 7: + return + if hasattr(line, 'decode'): # Python 2 + # The line could contain multi-byte characters + try: + length = len(line.decode('utf-8')) + except UnicodeError: + pass + if length > max_line_length: + return (max_line_length, "E501 line too long " + "(%d > %d characters)" % (length, max_line_length)) + + +############################################################################## +# Plugins (check functions) for logical lines +############################################################################## + + +def blank_lines(logical_line, blank_lines, indent_level, line_number, + blank_before, previous_logical, previous_indent_level): + r"""Separate top-level function and class definitions with two blank lines. + + Method definitions inside a class are separated by a single blank line. + + Extra blank lines may be used (sparingly) to separate groups of related + functions. Blank lines may be omitted between a bunch of related + one-liners (e.g. a set of dummy implementations). + + Use blank lines in functions, sparingly, to indicate logical sections. + + Okay: def a():\n pass\n\n\ndef b():\n pass + Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass + + E301: class Foo:\n b = 0\n def bar():\n pass + E302: def a():\n pass\n\ndef b(n):\n pass + E303: def a():\n pass\n\n\n\ndef b(n):\n pass + E303: def a():\n\n\n\n pass + E304: @decorator\n\ndef a():\n pass + """ + if line_number < 3 and not previous_logical: + return # Don't expect blank lines before the first line + if previous_logical.startswith('@'): + if blank_lines: + yield 0, "E304 blank lines found after function decorator" + elif blank_lines > 2 or (indent_level and blank_lines == 2): + yield 0, "E303 too many blank lines (%d)" % blank_lines + elif logical_line.startswith(('def ', 'class ', '@')): + if indent_level: + if not (blank_before or previous_indent_level < indent_level or + DOCSTRING_REGEX.match(previous_logical)): + yield 0, "E301 expected 1 blank line, found 0" + elif blank_before != 2: + yield 0, "E302 expected 2 blank lines, found %d" % blank_before + + +def extraneous_whitespace(logical_line): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in these situations: + - Immediately inside parentheses, brackets or braces. + - Immediately before a comma, semicolon, or colon. + + Okay: spam(ham[1], {eggs: 2}) + E201: spam( ham[1], {eggs: 2}) + E201: spam(ham[ 1], {eggs: 2}) + E201: spam(ham[1], { eggs: 2}) + E202: spam(ham[1], {eggs: 2} ) + E202: spam(ham[1 ], {eggs: 2}) + E202: spam(ham[1], {eggs: 2 }) + + E203: if x == 4: print x, y; x, y = y , x + E203: if x == 4: print x, y ; x, y = y, x + E203: if x == 4 : print x, y; x, y = y, x + """ + line = logical_line + for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): + text = match.group() + char = text.strip() + found = match.start() + if text == char + ' ': + # assert char in '([{' + yield found + 1, "E201 whitespace after '%s'" % char + elif line[found - 1] != ',': + code = ('E202' if char in '}])' else 'E203') # if char in ',;:' + yield found, "%s whitespace before '%s'" % (code, char) + + +def whitespace_around_keywords(logical_line): + r"""Avoid extraneous whitespace around keywords. + + Okay: True and False + E271: True and False + E272: True and False + E273: True and\tFalse + E274: True\tand False + """ + for match in KEYWORD_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E274 tab before keyword" + elif len(before) > 1: + yield match.start(1), "E272 multiple spaces before keyword" + + if '\t' in after: + yield match.start(2), "E273 tab after keyword" + elif len(after) > 1: + yield match.start(2), "E271 multiple spaces after keyword" + + +def missing_whitespace(logical_line): + r"""Each comma, semicolon or colon should be followed by whitespace. + + Okay: [a, b] + Okay: (3,) + Okay: a[1:4] + Okay: a[:4] + Okay: a[1:] + Okay: a[1:4:2] + E231: ['a','b'] + E231: foo(bar,baz) + E231: [{'a':'b'}] + """ + line = logical_line + for index in range(len(line) - 1): + char = line[index] + if char in ',;:' and line[index + 1] not in WHITESPACE: + before = line[:index] + if char == ':' and before.count('[') > before.count(']') and \ + before.rfind('{') < before.rfind('['): + continue # Slice syntax, no space required + if char == ',' and line[index + 1] == ')': + continue # Allow tuple with only one element: (3,) + yield index, "E231 missing whitespace after '%s'" % char + + +def indentation(logical_line, previous_logical, indent_char, + indent_level, previous_indent_level): + r"""Use 4 spaces per indentation level. + + For really old code that you don't want to mess up, you can continue to + use 8-space tabs. + + Okay: a = 1 + Okay: if a == 0:\n a = 1 + E111: a = 1 + E114: # a = 1 + + Okay: for item in items:\n pass + E112: for item in items:\npass + E115: for item in items:\n# Hi\n pass + + Okay: a = 1\nb = 2 + E113: a = 1\n b = 2 + E116: a = 1\n # b = 2 + """ + c = 0 if logical_line else 3 + tmpl = "E11%d %s" if logical_line else "E11%d %s (comment)" + if indent_level % 4: + yield 0, tmpl % (1 + c, "indentation is not a multiple of four") + indent_expect = previous_logical.endswith(':') + if indent_expect and indent_level <= previous_indent_level: + yield 0, tmpl % (2 + c, "expected an indented block") + elif not indent_expect and indent_level > previous_indent_level: + yield 0, tmpl % (3 + c, "unexpected indentation") + + +def continued_indentation(logical_line, tokens, indent_level, hang_closing, + indent_char, noqa, verbose): + r"""Continuation lines indentation. + + Continuation lines should align wrapped elements either vertically + using Python's implicit line joining inside parentheses, brackets + and braces, or using a hanging indent. + + When using a hanging indent these considerations should be applied: + - there should be no arguments on the first line, and + - further indentation should be used to clearly distinguish itself as a + continuation line. + + Okay: a = (\n) + E123: a = (\n ) + + Okay: a = (\n 42) + E121: a = (\n 42) + E122: a = (\n42) + E123: a = (\n 42\n ) + E124: a = (24,\n 42\n) + E125: if (\n b):\n pass + E126: a = (\n 42) + E127: a = (24,\n 42) + E128: a = (24,\n 42) + E129: if (a or\n b):\n pass + E131: a = (\n 42\n 24) + """ + first_row = tokens[0][2][0] + nrows = 1 + tokens[-1][2][0] - first_row + if noqa or nrows == 1: + return + + # indent_next tells us whether the next block is indented; assuming + # that it is indented by 4 spaces, then we should not allow 4-space + # indents on the final continuation line; in turn, some other + # indents are allowed to have an extra 4 spaces. + indent_next = logical_line.endswith(':') + + row = depth = 0 + valid_hangs = (4,) if indent_char != '\t' else (4, 8) + # remember how many brackets were opened on each line + parens = [0] * nrows + # relative indents of physical lines + rel_indent = [0] * nrows + # for each depth, collect a list of opening rows + open_rows = [[0]] + # for each depth, memorize the hanging indentation + hangs = [None] + # visual indents + indent_chances = {} + last_indent = tokens[0][2] + visual_indent = None + last_token_multiline = False + # for each depth, memorize the visual indent column + indent = [last_indent[1]] + if verbose >= 3: + print(">>> " + tokens[0][4].rstrip()) + + for token_type, text, start, end, line in tokens: + + newline = row < start[0] - first_row + if newline: + row = start[0] - first_row + newline = not last_token_multiline and token_type not in NEWLINE + + if newline: + # this is the beginning of a continuation line. + last_indent = start + if verbose >= 3: + print("... " + line.rstrip()) + + # record the initial indent. + rel_indent[row] = expand_indent(line) - indent_level + + # identify closing bracket + close_bracket = (token_type == tokenize.OP and text in ']})') + + # is the indent relative to an opening bracket line? + for open_row in reversed(open_rows[depth]): + hang = rel_indent[row] - rel_indent[open_row] + hanging_indent = hang in valid_hangs + if hanging_indent: + break + if hangs[depth]: + hanging_indent = (hang == hangs[depth]) + # is there any chance of visual indent? + visual_indent = (not close_bracket and hang > 0 and + indent_chances.get(start[1])) + + if close_bracket and indent[depth]: + # closing bracket for visual indent + if start[1] != indent[depth]: + yield (start, "E124 closing bracket does not match " + "visual indentation") + elif close_bracket and not hang: + # closing bracket matches indentation of opening bracket's line + if hang_closing: + yield start, "E133 closing bracket is missing indentation" + elif indent[depth] and start[1] < indent[depth]: + if visual_indent is not True: + # visual indent is broken + yield (start, "E128 continuation line " + "under-indented for visual indent") + elif hanging_indent or (indent_next and rel_indent[row] == 8): + # hanging indent is verified + if close_bracket and not hang_closing: + yield (start, "E123 closing bracket does not match " + "indentation of opening bracket's line") + hangs[depth] = hang + elif visual_indent is True: + # visual indent is verified + indent[depth] = start[1] + elif visual_indent in (text, str): + # ignore token lined up with matching one from a previous line + pass + else: + # indent is broken + if hang <= 0: + error = "E122", "missing indentation or outdented" + elif indent[depth]: + error = "E127", "over-indented for visual indent" + elif not close_bracket and hangs[depth]: + error = "E131", "unaligned for hanging indent" + else: + hangs[depth] = hang + if hang > 4: + error = "E126", "over-indented for hanging indent" + else: + error = "E121", "under-indented for hanging indent" + yield start, "%s continuation line %s" % error + + # look for visual indenting + if (parens[row] and + token_type not in (tokenize.NL, tokenize.COMMENT) and + not indent[depth]): + indent[depth] = start[1] + indent_chances[start[1]] = True + if verbose >= 4: + print("bracket depth %s indent to %s" % (depth, start[1])) + # deal with implicit string concatenation + elif (token_type in (tokenize.STRING, tokenize.COMMENT) or + text in ('u', 'ur', 'b', 'br')): + indent_chances[start[1]] = str + # special case for the "if" statement because len("if (") == 4 + elif not indent_chances and not row and not depth and text == 'if': + indent_chances[end[1] + 1] = True + elif text == ':' and line[end[1]:].isspace(): + open_rows[depth].append(row) + + # keep track of bracket depth + if token_type == tokenize.OP: + if text in '([{': + depth += 1 + indent.append(0) + hangs.append(None) + if len(open_rows) == depth: + open_rows.append([]) + open_rows[depth].append(row) + parens[row] += 1 + if verbose >= 4: + print("bracket depth %s seen, col %s, visual min = %s" % + (depth, start[1], indent[depth])) + elif text in ')]}' and depth > 0: + # parent indents should not be more than this one + prev_indent = indent.pop() or last_indent[1] + hangs.pop() + for d in range(depth): + if indent[d] > prev_indent: + indent[d] = 0 + for ind in list(indent_chances): + if ind >= prev_indent: + del indent_chances[ind] + del open_rows[depth + 1:] + depth -= 1 + if depth: + indent_chances[indent[depth]] = True + for idx in range(row, -1, -1): + if parens[idx]: + parens[idx] -= 1 + break + assert len(indent) == depth + 1 + if start[1] not in indent_chances: + # allow to line up tokens + indent_chances[start[1]] = text + + last_token_multiline = (start[0] != end[0]) + if last_token_multiline: + rel_indent[end[0] - first_row] = rel_indent[row] + + if indent_next and expand_indent(line) == indent_level + 4: + pos = (start[0], indent[0] + 4) + if visual_indent: + code = "E129 visually indented line" + else: + code = "E125 continuation line" + yield pos, "%s with same indent as next logical line" % code + + +def whitespace_before_parameters(logical_line, tokens): + r"""Avoid extraneous whitespace. + + Avoid extraneous whitespace in the following situations: + - before the open parenthesis that starts the argument list of a + function call. + - before the open parenthesis that starts an indexing or slicing. + + Okay: spam(1) + E211: spam (1) + + Okay: dict['key'] = list[index] + E211: dict ['key'] = list[index] + E211: dict['key'] = list [index] + """ + prev_type, prev_text, __, prev_end, __ = tokens[0] + for index in range(1, len(tokens)): + token_type, text, start, end, __ = tokens[index] + if (token_type == tokenize.OP and + text in '([' and + start != prev_end and + (prev_type == tokenize.NAME or prev_text in '}])') and + # Syntax "class A (B):" is allowed, but avoid it + (index < 2 or tokens[index - 2][1] != 'class') and + # Allow "return (a.foo for a in range(5))" + not keyword.iskeyword(prev_text)): + yield prev_end, "E211 whitespace before '%s'" % text + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_operator(logical_line): + r"""Avoid extraneous whitespace around an operator. + + Okay: a = 12 + 3 + E221: a = 4 + 5 + E222: a = 4 + 5 + E223: a = 4\t+ 5 + E224: a = 4 +\t5 + """ + for match in OPERATOR_REGEX.finditer(logical_line): + before, after = match.groups() + + if '\t' in before: + yield match.start(1), "E223 tab before operator" + elif len(before) > 1: + yield match.start(1), "E221 multiple spaces before operator" + + if '\t' in after: + yield match.start(2), "E224 tab after operator" + elif len(after) > 1: + yield match.start(2), "E222 multiple spaces after operator" + + +def missing_whitespace_around_operator(logical_line, tokens): + r"""Surround operators with a single space on either side. + + - Always surround these binary operators with a single space on + either side: assignment (=), augmented assignment (+=, -= etc.), + comparisons (==, <, >, !=, <=, >=, in, not in, is, is not), + Booleans (and, or, not). + + - If operators with different priorities are used, consider adding + whitespace around the operators with the lowest priorities. + + Okay: i = i + 1 + Okay: submitted += 1 + Okay: x = x * 2 - 1 + Okay: hypot2 = x * x + y * y + Okay: c = (a + b) * (a - b) + Okay: foo(bar, key='word', *args, **kwargs) + Okay: alpha[:-i] + + E225: i=i+1 + E225: submitted +=1 + E225: x = x /2 - 1 + E225: z = x **y + E226: c = (a+b) * (a-b) + E226: hypot2 = x*x + y*y + E227: c = a|b + E228: msg = fmt%(errno, errmsg) + """ + parens = 0 + need_space = False + prev_type = tokenize.OP + prev_text = prev_end = None + for token_type, text, start, end, line in tokens: + if token_type in SKIP_COMMENTS: + continue + if text in ('(', 'lambda'): + parens += 1 + elif text == ')': + parens -= 1 + if need_space: + if start != prev_end: + # Found a (probably) needed space + if need_space is not True and not need_space[1]: + yield (need_space[0], + "E225 missing whitespace around operator") + need_space = False + elif text == '>' and prev_text in ('<', '-'): + # Tolerate the "<>" operator, even if running Python 3 + # Deal with Python 3's annotated return value "->" + pass + else: + if need_space is True or need_space[1]: + # A needed trailing space was not found + yield prev_end, "E225 missing whitespace around operator" + elif prev_text != '**': + code, optype = 'E226', 'arithmetic' + if prev_text == '%': + code, optype = 'E228', 'modulo' + elif prev_text not in ARITHMETIC_OP: + code, optype = 'E227', 'bitwise or shift' + yield (need_space[0], "%s missing whitespace " + "around %s operator" % (code, optype)) + need_space = False + elif token_type == tokenize.OP and prev_end is not None: + if text == '=' and parens: + # Allow keyword args or defaults: foo(bar=None). + pass + elif text in WS_NEEDED_OPERATORS: + need_space = True + elif text in UNARY_OPERATORS: + # Check if the operator is being used as a binary operator + # Allow unary operators: -123, -x, +1. + # Allow argument unpacking: foo(*args, **kwargs). + if (prev_text in '}])' if prev_type == tokenize.OP + else prev_text not in KEYWORDS): + need_space = None + elif text in WS_OPTIONAL_OPERATORS: + need_space = None + + if need_space is None: + # Surrounding space is optional, but ensure that + # trailing space matches opening space + need_space = (prev_end, start != prev_end) + elif need_space and start == prev_end: + # A needed opening space was not found + yield prev_end, "E225 missing whitespace around operator" + need_space = False + prev_type = token_type + prev_text = text + prev_end = end + + +def whitespace_around_comma(logical_line): + r"""Avoid extraneous whitespace after a comma or a colon. + + Note: these checks are disabled by default + + Okay: a = (1, 2) + E241: a = (1, 2) + E242: a = (1,\t2) + """ + line = logical_line + for m in WHITESPACE_AFTER_COMMA_REGEX.finditer(line): + found = m.start() + 1 + if '\t' in m.group(): + yield found, "E242 tab after '%s'" % m.group()[0] + else: + yield found, "E241 multiple spaces after '%s'" % m.group()[0] + + +def whitespace_around_named_parameter_equals(logical_line, tokens): + r"""Don't use spaces around the '=' sign in function arguments. + + Don't use spaces around the '=' sign when used to indicate a + keyword argument or a default parameter value. + + Okay: def complex(real, imag=0.0): + Okay: return magic(r=real, i=imag) + Okay: boolean(a == b) + Okay: boolean(a != b) + Okay: boolean(a <= b) + Okay: boolean(a >= b) + Okay: def foo(arg: int = 42): + + E251: def complex(real, imag = 0.0): + E251: return magic(r = real, i = imag) + """ + parens = 0 + no_space = False + prev_end = None + annotated_func_arg = False + in_def = logical_line.startswith('def') + message = "E251 unexpected spaces around keyword / parameter equals" + for token_type, text, start, end, line in tokens: + if token_type == tokenize.NL: + continue + if no_space: + no_space = False + if start != prev_end: + yield (prev_end, message) + if token_type == tokenize.OP: + if text == '(': + parens += 1 + elif text == ')': + parens -= 1 + elif in_def and text == ':' and parens == 1: + annotated_func_arg = True + elif parens and text == ',' and parens == 1: + annotated_func_arg = False + elif parens and text == '=' and not annotated_func_arg: + no_space = True + if start != prev_end: + yield (prev_end, message) + if not parens: + annotated_func_arg = False + + prev_end = end + + +def whitespace_before_comment(logical_line, tokens): + r"""Separate inline comments by at least two spaces. + + An inline comment is a comment on the same line as a statement. Inline + comments should be separated by at least two spaces from the statement. + They should start with a # and a single space. + + Each line of a block comment starts with a # and a single space + (unless it is indented text inside the comment). + + Okay: x = x + 1 # Increment x + Okay: x = x + 1 # Increment x + Okay: # Block comment + E261: x = x + 1 # Increment x + E262: x = x + 1 #Increment x + E262: x = x + 1 # Increment x + E265: #Block comment + E266: ### Block comment + """ + prev_end = (0, 0) + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + inline_comment = line[:start[1]].strip() + if inline_comment: + if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: + yield (prev_end, + "E261 at least two spaces before inline comment") + symbol, sp, comment = text.partition(' ') + bad_prefix = symbol not in '#:' and (symbol.lstrip('#')[:1] or '#') + if inline_comment: + if bad_prefix or comment[:1] in WHITESPACE: + yield start, "E262 inline comment should start with '# '" + elif bad_prefix and (bad_prefix != '!' or start[0] > 1): + if bad_prefix != '#': + yield start, "E265 block comment should start with '# '" + elif comment: + yield start, "E266 too many leading '#' for block comment" + elif token_type != tokenize.NL: + prev_end = end + + +def imports_on_separate_lines(logical_line): + r"""Imports should usually be on separate lines. + + Okay: import os\nimport sys + E401: import sys, os + + Okay: from subprocess import Popen, PIPE + Okay: from myclas import MyClass + Okay: from foo.bar.yourclass import YourClass + Okay: import myclass + Okay: import foo.bar.yourclass + """ + line = logical_line + if line.startswith('import '): + found = line.find(',') + if -1 < found and ';' not in line[:found]: + yield found, "E401 multiple imports on one line" + + +def module_imports_on_top_of_file( + logical_line, indent_level, checker_state, noqa): + r"""Imports are always put at the top of the file, just after any module + comments and docstrings, and before module globals and constants. + + Okay: import os + Okay: # this is a comment\nimport os + Okay: '''this is a module docstring'''\nimport os + Okay: r'''this is a module docstring'''\nimport os + Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y + Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y + E402: a=1\nimport os + E402: 'One string'\n"Two string"\nimport os + E402: a=1\nfrom sys import x + + Okay: if x:\n import os + """ + def is_string_literal(line): + if line[0] in 'uUbB': + line = line[1:] + if line and line[0] in 'rR': + line = line[1:] + return line and (line[0] == '"' or line[0] == "'") + + allowed_try_keywords = ('try', 'except', 'else', 'finally') + + if indent_level: # Allow imports in conditional statements or functions + return + if not logical_line: # Allow empty lines or comments + return + if noqa: + return + line = logical_line + if line.startswith('import ') or line.startswith('from '): + if checker_state.get('seen_non_imports', False): + yield 0, "E402 module level import not at top of file" + elif any(line.startswith(kw) for kw in allowed_try_keywords): + # Allow try, except, else, finally keywords intermixed with imports in + # order to support conditional importing + return + elif is_string_literal(line): + # The first literal is a docstring, allow it. Otherwise, report error. + if checker_state.get('seen_docstring', False): + checker_state['seen_non_imports'] = True + else: + checker_state['seen_docstring'] = True + else: + checker_state['seen_non_imports'] = True + + +def compound_statements(logical_line): + r"""Compound statements (on the same line) are generally discouraged. + + While sometimes it's okay to put an if/for/while with a small body + on the same line, never do this for multi-clause statements. + Also avoid folding such long lines! + + Always use a def statement instead of an assignment statement that + binds a lambda expression directly to a name. + + Okay: if foo == 'blah':\n do_blah_thing() + Okay: do_one() + Okay: do_two() + Okay: do_three() + + E701: if foo == 'blah': do_blah_thing() + E701: for x in lst: total += x + E701: while t < 10: t = delay() + E701: if foo == 'blah': do_blah_thing() + E701: else: do_non_blah_thing() + E701: try: something() + E701: finally: cleanup() + E701: if foo == 'blah': one(); two(); three() + E702: do_one(); do_two(); do_three() + E703: do_four(); # useless semicolon + E704: def f(x): return 2*x + E731: f = lambda x: 2*x + """ + line = logical_line + last_char = len(line) - 1 + found = line.find(':') + while -1 < found < last_char: + before = line[:found] + if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) + before.count('[') <= before.count(']') and # [1:2] (slice) + before.count('(') <= before.count(')'))): # (annotation) + lambda_kw = LAMBDA_REGEX.search(before) + if lambda_kw: + before = line[:lambda_kw.start()].rstrip() + if before[-1:] == '=' and isidentifier(before[:-1].strip()): + yield 0, ("E731 do not assign a lambda expression, use a " + "def") + break + if before.startswith('def '): + yield 0, "E704 multiple statements on one line (def)" + else: + yield found, "E701 multiple statements on one line (colon)" + found = line.find(':', found + 1) + found = line.find(';') + while -1 < found: + if found < last_char: + yield found, "E702 multiple statements on one line (semicolon)" + else: + yield found, "E703 statement ends with a semicolon" + found = line.find(';', found + 1) + + +def explicit_line_join(logical_line, tokens): + r"""Avoid explicit line join between brackets. + + The preferred way of wrapping long lines is by using Python's implied line + continuation inside parentheses, brackets and braces. Long lines can be + broken over multiple lines by wrapping expressions in parentheses. These + should be used in preference to using a backslash for line continuation. + + E502: aaa = [123, \\n 123] + E502: aaa = ("bbb " \\n "ccc") + + Okay: aaa = [123,\n 123] + Okay: aaa = ("bbb "\n "ccc") + Okay: aaa = "bbb " \\n "ccc" + Okay: aaa = 123 # \\ + """ + prev_start = prev_end = parens = 0 + comment = False + backslash = None + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + comment = True + if start[0] != prev_start and parens and backslash and not comment: + yield backslash, "E502 the backslash is redundant between brackets" + if end[0] != prev_end: + if line.rstrip('\r\n').endswith('\\'): + backslash = (end[0], len(line.splitlines()[-1]) - 1) + else: + backslash = None + prev_start = prev_end = end[0] + else: + prev_start = start[0] + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in ')]}': + parens -= 1 + + +def break_around_binary_operator(logical_line, tokens): + r""" + Avoid breaks before binary operators. + + The preferred place to break around a binary operator is after the + operator, not before it. + + W503: (width == 0\n + height == 0) + W503: (width == 0\n and height == 0) + + Okay: (width == 0 +\n height == 0) + Okay: foo(\n -x) + Okay: foo(x\n []) + Okay: x = '''\n''' + '' + Okay: foo(x,\n -y) + Okay: foo(x, # comment\n -y) + """ + def is_binary_operator(token_type, text): + # The % character is strictly speaking a binary operator, but the + # common usage seems to be to put it next to the format parameters, + # after a line break. + return ((token_type == tokenize.OP or text in ['and', 'or']) and + text not in "()[]{},:.;@=%") + + line_break = False + unary_context = True + for token_type, text, start, end, line in tokens: + if token_type == tokenize.COMMENT: + continue + if ('\n' in text or '\r' in text) and token_type != tokenize.STRING: + line_break = True + else: + if (is_binary_operator(token_type, text) and line_break and + not unary_context): + yield start, "W503 line break before binary operator" + unary_context = text in '([{,;' + line_break = False + + +def comparison_to_singleton(logical_line, noqa): + r"""Comparison to singletons should use "is" or "is not". + + Comparisons to singletons like None should always be done + with "is" or "is not", never the equality operators. + + Okay: if arg is not None: + E711: if arg != None: + E711: if None == arg: + E712: if arg == True: + E712: if False == arg: + + Also, beware of writing if x when you really mean if x is not None -- + e.g. when testing whether a variable or argument that defaults to None was + set to some other value. The other value might have a type (such as a + container) that could be false in a boolean context! + """ + match = not noqa and COMPARE_SINGLETON_REGEX.search(logical_line) + if match: + singleton = match.group(1) or match.group(3) + same = (match.group(2) == '==') + + msg = "'if cond is %s:'" % (('' if same else 'not ') + singleton) + if singleton in ('None',): + code = 'E711' + else: + code = 'E712' + nonzero = ((singleton == 'True' and same) or + (singleton == 'False' and not same)) + msg += " or 'if %scond:'" % ('' if nonzero else 'not ') + yield match.start(2), ("%s comparison to %s should be %s" % + (code, singleton, msg)) + + +def comparison_negative(logical_line): + r"""Negative comparison should be done using "not in" and "is not". + + Okay: if x not in y:\n pass + Okay: assert (X in Y or X is Z) + Okay: if not (X in Y):\n pass + Okay: zz = x is not y + E713: Z = not X in Y + E713: if not X.B in Y:\n pass + E714: if not X is Y:\n pass + E714: Z = not X.B is Y + """ + match = COMPARE_NEGATIVE_REGEX.search(logical_line) + if match: + pos = match.start(1) + if match.group(2) == 'in': + yield pos, "E713 test for membership should be 'not in'" + else: + yield pos, "E714 test for object identity should be 'is not'" + + +def comparison_type(logical_line, noqa): + r"""Object type comparisons should always use isinstance(). + + Do not compare types directly. + + Okay: if isinstance(obj, int): + E721: if type(obj) is type(1): + + When checking if an object is a string, keep in mind that it might be a + unicode string too! In Python 2.3, str and unicode have a common base + class, basestring, so you can do: + + Okay: if isinstance(obj, basestring): + Okay: if type(a1) is type(b1): + """ + match = COMPARE_TYPE_REGEX.search(logical_line) + if match and not noqa: + inst = match.group(1) + if inst and isidentifier(inst) and inst not in SINGLETONS: + return # Allow comparison for types which are not obvious + yield match.start(), "E721 do not compare types, use 'isinstance()'" + + +def python_3000_has_key(logical_line, noqa): + r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. + + Okay: if "alph" in d:\n print d["alph"] + W601: assert d.has_key('alph') + """ + pos = logical_line.find('.has_key(') + if pos > -1 and not noqa: + yield pos, "W601 .has_key() is deprecated, use 'in'" + + +def python_3000_raise_comma(logical_line): + r"""When raising an exception, use "raise ValueError('message')". + + The older form is removed in Python 3. + + Okay: raise DummyError("Message") + W602: raise DummyError, "Message" + """ + match = RAISE_COMMA_REGEX.match(logical_line) + if match and not RERAISE_COMMA_REGEX.match(logical_line): + yield match.end() - 1, "W602 deprecated form of raising exception" + + +def python_3000_not_equal(logical_line): + r"""New code should always use != instead of <>. + + The older syntax is removed in Python 3. + + Okay: if a != 'no': + W603: if a <> 'no': + """ + pos = logical_line.find('<>') + if pos > -1: + yield pos, "W603 '<>' is deprecated, use '!='" + + +def python_3000_backticks(logical_line): + r"""Backticks are removed in Python 3: use repr() instead. + + Okay: val = repr(1 + 2) + W604: val = `1 + 2` + """ + pos = logical_line.find('`') + if pos > -1: + yield pos, "W604 backticks are deprecated, use 'repr()'" + + +############################################################################## +# Helper functions +############################################################################## + + +if sys.version_info < (3,): + # Python 2: implicit encoding. + def readlines(filename): + """Read the source code.""" + with open(filename, 'rU') as f: + return f.readlines() + isidentifier = re.compile(r'[a-zA-Z_]\w*$').match + stdin_get_value = sys.stdin.read +else: + # Python 3 + def readlines(filename): + """Read the source code.""" + try: + with open(filename, 'rb') as f: + (coding, lines) = tokenize.detect_encoding(f.readline) + f = TextIOWrapper(f, coding, line_buffering=True) + return [l.decode(coding) for l in lines] + f.readlines() + except (LookupError, SyntaxError, UnicodeError): + # Fall back if file encoding is improperly declared + with open(filename, encoding='latin-1') as f: + return f.readlines() + isidentifier = str.isidentifier + + def stdin_get_value(): + return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() +noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search + + +def expand_indent(line): + r"""Return the amount of indentation. + + Tabs are expanded to the next multiple of 8. + + >>> expand_indent(' ') + 4 + >>> expand_indent('\t') + 8 + >>> expand_indent(' \t') + 8 + >>> expand_indent(' \t') + 16 + """ + if '\t' not in line: + return len(line) - len(line.lstrip()) + result = 0 + for char in line: + if char == '\t': + result = result // 8 * 8 + 8 + elif char == ' ': + result += 1 + else: + break + return result + + +def mute_string(text): + """Replace contents with 'xxx' to prevent syntax matching. + + >>> mute_string('"abc"') + '"xxx"' + >>> mute_string("'''abc'''") + "'''xxx'''" + >>> mute_string("r'abc'") + "r'xxx'" + """ + # String modifiers (e.g. u or r) + start = text.index(text[-1]) + 1 + end = len(text) - 1 + # Triple quotes + if text[-3:] in ('"""', "'''"): + start += 2 + end -= 2 + return text[:start] + 'x' * (end - start) + text[end:] + + +def parse_udiff(diff, patterns=None, parent='.'): + """Return a dictionary of matching lines.""" + # For each file of the diff, the entry key is the filename, + # and the value is a set of row numbers to consider. + rv = {} + path = nrows = None + for line in diff.splitlines(): + if nrows: + if line[:1] != '-': + nrows -= 1 + continue + if line[:3] == '@@ ': + hunk_match = HUNK_REGEX.match(line) + (row, nrows) = [int(g or '1') for g in hunk_match.groups()] + rv[path].update(range(row, row + nrows)) + elif line[:3] == '+++': + path = line[4:].split('\t', 1)[0] + if path[:2] == 'b/': + path = path[2:] + rv[path] = set() + return dict([(os.path.join(parent, path), rows) + for (path, rows) in rv.items() + if rows and filename_match(path, patterns)]) + + +def normalize_paths(value, parent=os.curdir): + """Parse a comma-separated list of paths. + + Return a list of absolute paths. + """ + if not value: + return [] + if isinstance(value, list): + return value + paths = [] + for path in value.split(','): + path = path.strip() + if '/' in path: + path = os.path.abspath(os.path.join(parent, path)) + paths.append(path.rstrip('/')) + return paths + + +def filename_match(filename, patterns, default=True): + """Check if patterns contains a pattern that matches filename. + + If patterns is unspecified, this always returns True. + """ + if not patterns: + return default + return any(fnmatch(filename, pattern) for pattern in patterns) + + +def _is_eol_token(token): + return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' +if COMMENT_WITH_NL: + def _is_eol_token(token, _eol_token=_is_eol_token): + return _eol_token(token) or (token[0] == tokenize.COMMENT and + token[1] == token[4]) + +############################################################################## +# Framework to run all checks +############################################################################## + + +_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} + + +def _get_parameters(function): + if sys.version_info >= (3, 3): + return [parameter.name + for parameter + in inspect.signature(function).parameters.values() + if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] + else: + return inspect.getargspec(function)[0] + + +def register_check(check, codes=None): + """Register a new check object.""" + def _add_check(check, kind, codes, args): + if check in _checks[kind]: + _checks[kind][check][0].extend(codes or []) + else: + _checks[kind][check] = (codes or [''], args) + if inspect.isfunction(check): + args = _get_parameters(check) + if args and args[0] in ('physical_line', 'logical_line'): + if codes is None: + codes = ERRORCODE_REGEX.findall(check.__doc__ or '') + _add_check(check, args[0], codes, args) + elif inspect.isclass(check): + if _get_parameters(check.__init__)[:2] == ['self', 'tree']: + _add_check(check, 'tree', codes, None) + + +def init_checks_registry(): + """Register all globally visible functions. + + The first argument name is either 'physical_line' or 'logical_line'. + """ + mod = inspect.getmodule(register_check) + for (name, function) in inspect.getmembers(mod, inspect.isfunction): + register_check(function) +init_checks_registry() + + +class Checker(object): + """Load a Python source file, tokenize it, check coding style.""" + + def __init__(self, filename=None, lines=None, + options=None, report=None, **kwargs): + if options is None: + options = StyleGuide(kwargs).options + else: + assert not kwargs + self._io_error = None + self._physical_checks = options.physical_checks + self._logical_checks = options.logical_checks + self._ast_checks = options.ast_checks + self.max_line_length = options.max_line_length + self.multiline = False # in a multiline string? + self.hang_closing = options.hang_closing + self.verbose = options.verbose + self.filename = filename + # Dictionary where a checker can store its custom state. + self._checker_states = {} + if filename is None: + self.filename = 'stdin' + self.lines = lines or [] + elif filename == '-': + self.filename = 'stdin' + self.lines = stdin_get_value().splitlines(True) + elif lines is None: + try: + self.lines = readlines(filename) + except IOError: + (exc_type, exc) = sys.exc_info()[:2] + self._io_error = '%s: %s' % (exc_type.__name__, exc) + self.lines = [] + else: + self.lines = lines + if self.lines: + ord0 = ord(self.lines[0][0]) + if ord0 in (0xef, 0xfeff): # Strip the UTF-8 BOM + if ord0 == 0xfeff: + self.lines[0] = self.lines[0][1:] + elif self.lines[0][:3] == '\xef\xbb\xbf': + self.lines[0] = self.lines[0][3:] + self.report = report or options.report + self.report_error = self.report.error + + def report_invalid_syntax(self): + """Check if the syntax is valid.""" + (exc_type, exc) = sys.exc_info()[:2] + if len(exc.args) > 1: + offset = exc.args[1] + if len(offset) > 2: + offset = offset[1:3] + else: + offset = (1, 0) + self.report_error(offset[0], offset[1] or 0, + 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), + self.report_invalid_syntax) + + def readline(self): + """Get the next line from the input buffer.""" + if self.line_number >= self.total_lines: + return '' + line = self.lines[self.line_number] + self.line_number += 1 + if self.indent_char is None and line[:1] in WHITESPACE: + self.indent_char = line[0] + return line + + def run_check(self, check, argument_names): + """Run a check plugin.""" + arguments = [] + for name in argument_names: + arguments.append(getattr(self, name)) + return check(*arguments) + + def init_checker_state(self, name, argument_names): + """ Prepares a custom state for the specific checker plugin.""" + if 'checker_state' in argument_names: + self.checker_state = self._checker_states.setdefault(name, {}) + + def check_physical(self, line): + """Run all physical checks on a raw input line.""" + self.physical_line = line + for name, check, argument_names in self._physical_checks: + self.init_checker_state(name, argument_names) + result = self.run_check(check, argument_names) + if result is not None: + (offset, text) = result + self.report_error(self.line_number, offset, text, check) + if text[:4] == 'E101': + self.indent_char = line[0] + + def build_tokens_line(self): + """Build a logical line from tokens.""" + logical = [] + comments = [] + length = 0 + prev_row = prev_col = mapping = None + for token_type, text, start, end, line in self.tokens: + if token_type in SKIP_TOKENS: + continue + if not mapping: + mapping = [(0, start)] + if token_type == tokenize.COMMENT: + comments.append(text) + continue + if token_type == tokenize.STRING: + text = mute_string(text) + if prev_row: + (start_row, start_col) = start + if prev_row != start_row: # different row + prev_text = self.lines[prev_row - 1][prev_col - 1] + if prev_text == ',' or (prev_text not in '{[(' and + text not in '}])'): + text = ' ' + text + elif prev_col != start_col: # different column + text = line[prev_col:start_col] + text + logical.append(text) + length += len(text) + mapping.append((length, end)) + (prev_row, prev_col) = end + self.logical_line = ''.join(logical) + self.noqa = comments and noqa(''.join(comments)) + return mapping + + def check_logical(self): + """Build a line from tokens and run all logical checks on it.""" + self.report.increment_logical_line() + mapping = self.build_tokens_line() + + if not mapping: + return + + (start_row, start_col) = mapping[0][1] + start_line = self.lines[start_row - 1] + self.indent_level = expand_indent(start_line[:start_col]) + if self.blank_before < self.blank_lines: + self.blank_before = self.blank_lines + if self.verbose >= 2: + print(self.logical_line[:80].rstrip()) + for name, check, argument_names in self._logical_checks: + if self.verbose >= 4: + print(' ' + name) + self.init_checker_state(name, argument_names) + for offset, text in self.run_check(check, argument_names) or (): + if not isinstance(offset, tuple): + for token_offset, pos in mapping: + if offset <= token_offset: + break + offset = (pos[0], pos[1] + offset - token_offset) + self.report_error(offset[0], offset[1], text, check) + if self.logical_line: + self.previous_indent_level = self.indent_level + self.previous_logical = self.logical_line + self.blank_lines = 0 + self.tokens = [] + + def check_ast(self): + """Build the file's AST and run all AST checks.""" + try: + tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) + except (ValueError, SyntaxError, TypeError): + return self.report_invalid_syntax() + for name, cls, __ in self._ast_checks: + checker = cls(tree, self.filename) + for lineno, offset, text, check in checker.run(): + if not self.lines or not noqa(self.lines[lineno - 1]): + self.report_error(lineno, offset, text, check) + + def generate_tokens(self): + """Tokenize the file, run physical line checks and yield tokens.""" + if self._io_error: + self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) + tokengen = tokenize.generate_tokens(self.readline) + try: + for token in tokengen: + if token[2][0] > self.total_lines: + return + self.maybe_check_physical(token) + yield token + except (SyntaxError, tokenize.TokenError): + self.report_invalid_syntax() + + def maybe_check_physical(self, token): + """If appropriate (based on token), check current physical line(s).""" + # Called after every token, but act only on end of line. + if _is_eol_token(token): + # Obviously, a newline token ends a single physical line. + self.check_physical(token[4]) + elif token[0] == tokenize.STRING and '\n' in token[1]: + # Less obviously, a string that contains newlines is a + # multiline string, either triple-quoted or with internal + # newlines backslash-escaped. Check every physical line in the + # string *except* for the last one: its newline is outside of + # the multiline string, so we consider it a regular physical + # line, and will check it like any other physical line. + # + # Subtleties: + # - we don't *completely* ignore the last line; if it contains + # the magical "# noqa" comment, we disable all physical + # checks for the entire multiline string + # - have to wind self.line_number back because initially it + # points to the last line of the string, and we want + # check_physical() to give accurate feedback + if noqa(token[4]): + return + self.multiline = True + self.line_number = token[2][0] + for line in token[1].split('\n')[:-1]: + self.check_physical(line + '\n') + self.line_number += 1 + self.multiline = False + + def check_all(self, expected=None, line_offset=0): + """Run all checks on the input file.""" + self.report.init_file(self.filename, self.lines, expected, line_offset) + self.total_lines = len(self.lines) + if self._ast_checks: + self.check_ast() + self.line_number = 0 + self.indent_char = None + self.indent_level = self.previous_indent_level = 0 + self.previous_logical = '' + self.tokens = [] + self.blank_lines = self.blank_before = 0 + parens = 0 + for token in self.generate_tokens(): + self.tokens.append(token) + token_type, text = token[0:2] + if self.verbose >= 3: + if token[2][0] == token[3][0]: + pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) + else: + pos = 'l.%s' % token[3][0] + print('l.%s\t%s\t%s\t%r' % + (token[2][0], pos, tokenize.tok_name[token[0]], text)) + if token_type == tokenize.OP: + if text in '([{': + parens += 1 + elif text in '}])': + parens -= 1 + elif not parens: + if token_type in NEWLINE: + if token_type == tokenize.NEWLINE: + self.check_logical() + self.blank_before = 0 + elif len(self.tokens) == 1: + # The physical line contains only this token. + self.blank_lines += 1 + del self.tokens[0] + else: + self.check_logical() + elif COMMENT_WITH_NL and token_type == tokenize.COMMENT: + if len(self.tokens) == 1: + # The comment also ends a physical line + token = list(token) + token[1] = text.rstrip('\r\n') + token[3] = (token[2][0], token[2][1] + len(token[1])) + self.tokens = [tuple(token)] + self.check_logical() + if self.tokens: + self.check_physical(self.lines[-1]) + self.check_logical() + return self.report.get_file_results() + + +class BaseReport(object): + """Collect the results of the checks.""" + + print_filename = False + + def __init__(self, options): + self._benchmark_keys = options.benchmark_keys + self._ignore_code = options.ignore_code + # Results + self.elapsed = 0 + self.total_errors = 0 + self.counters = dict.fromkeys(self._benchmark_keys, 0) + self.messages = {} + + def start(self): + """Start the timer.""" + self._start_time = time.time() + + def stop(self): + """Stop the timer.""" + self.elapsed = time.time() - self._start_time + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self.filename = filename + self.lines = lines + self.expected = expected or () + self.line_offset = line_offset + self.file_errors = 0 + self.counters['files'] += 1 + self.counters['physical lines'] += len(lines) + + def increment_logical_line(self): + """Signal a new logical line.""" + self.counters['logical lines'] += 1 + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = text[:4] + if self._ignore_code(code): + return + if code in self.counters: + self.counters[code] += 1 + else: + self.counters[code] = 1 + self.messages[code] = text[5:] + # Don't care about expected errors or warnings + if code in self.expected: + return + if self.print_filename and not self.file_errors: + print(self.filename) + self.file_errors += 1 + self.total_errors += 1 + return code + + def get_file_results(self): + """Return the count of errors and warnings for this file.""" + return self.file_errors + + def get_count(self, prefix=''): + """Return the total count of errors and warnings.""" + return sum([self.counters[key] + for key in self.messages if key.startswith(prefix)]) + + def get_statistics(self, prefix=''): + """Get statistics for message codes that start with the prefix. + + prefix='' matches all errors and warnings + prefix='E' matches all errors + prefix='W' matches all warnings + prefix='E4' matches all errors that have to do with imports + """ + return ['%-7s %s %s' % (self.counters[key], key, self.messages[key]) + for key in sorted(self.messages) if key.startswith(prefix)] + + def print_statistics(self, prefix=''): + """Print overall statistics (number of errors and warnings).""" + for line in self.get_statistics(prefix): + print(line) + + def print_benchmark(self): + """Print benchmark numbers.""" + print('%-7.2f %s' % (self.elapsed, 'seconds elapsed')) + if self.elapsed: + for key in self._benchmark_keys: + print('%-7d %s per second (%d total)' % + (self.counters[key] / self.elapsed, key, + self.counters[key])) + + +class FileReport(BaseReport): + """Collect the results of the checks and print only the filenames.""" + print_filename = True + + +class StandardReport(BaseReport): + """Collect and print the results of the checks.""" + + def __init__(self, options): + super(StandardReport, self).__init__(options) + self._fmt = REPORT_FORMAT.get(options.format.lower(), + options.format) + self._repeat = options.repeat + self._show_source = options.show_source + self._show_pep8 = options.show_pep8 + + def init_file(self, filename, lines, expected, line_offset): + """Signal a new file.""" + self._deferred_print = [] + return super(StandardReport, self).init_file( + filename, lines, expected, line_offset) + + def error(self, line_number, offset, text, check): + """Report an error, according to options.""" + code = super(StandardReport, self).error(line_number, offset, + text, check) + if code and (self.counters[code] == 1 or self._repeat): + self._deferred_print.append( + (line_number, offset, code, text[5:], check.__doc__)) + return code + + def get_file_results(self): + """Print the result and return the overall count for this file.""" + self._deferred_print.sort() + for line_number, offset, code, text, doc in self._deferred_print: + print(self._fmt % { + 'path': self.filename, + 'row': self.line_offset + line_number, 'col': offset + 1, + 'code': code, 'text': text, + }) + if self._show_source: + if line_number > len(self.lines): + line = '' + else: + line = self.lines[line_number - 1] + print(line.rstrip()) + print(re.sub(r'\S', ' ', line[:offset]) + '^') + if self._show_pep8 and doc: + print(' ' + doc.strip()) + + # stdout is block buffered when not stdout.isatty(). + # line can be broken where buffer boundary since other processes + # write to same file. + # flush() after print() to avoid buffer boundary. + # Typical buffer size is 8192. line written safely when + # len(line) < 8192. + sys.stdout.flush() + return self.file_errors + + +class DiffReport(StandardReport): + """Collect and print the results for the changed lines only.""" + + def __init__(self, options): + super(DiffReport, self).__init__(options) + self._selected = options.selected_lines + + def error(self, line_number, offset, text, check): + if line_number not in self._selected[self.filename]: + return + return super(DiffReport, self).error(line_number, offset, text, check) + + +class StyleGuide(object): + """Initialize a PEP-8 instance with few options.""" + + def __init__(self, *args, **kwargs): + # build options from the command line + self.checker_class = kwargs.pop('checker_class', Checker) + parse_argv = kwargs.pop('parse_argv', False) + config_file = kwargs.pop('config_file', False) + parser = kwargs.pop('parser', None) + # build options from dict + options_dict = dict(*args, **kwargs) + arglist = None if parse_argv else options_dict.get('paths', None) + options, self.paths = process_options( + arglist, parse_argv, config_file, parser) + if options_dict: + options.__dict__.update(options_dict) + if 'paths' in options_dict: + self.paths = options_dict['paths'] + + self.runner = self.input_file + self.options = options + + if not options.reporter: + options.reporter = BaseReport if options.quiet else StandardReport + + options.select = tuple(options.select or ()) + if not (options.select or options.ignore or + options.testsuite or options.doctest) and DEFAULT_IGNORE: + # The default choice: ignore controversial checks + options.ignore = tuple(DEFAULT_IGNORE.split(',')) + else: + # Ignore all checks which are not explicitly selected + options.ignore = ('',) if options.select else tuple(options.ignore) + options.benchmark_keys = BENCHMARK_KEYS[:] + options.ignore_code = self.ignore_code + options.physical_checks = self.get_checks('physical_line') + options.logical_checks = self.get_checks('logical_line') + options.ast_checks = self.get_checks('tree') + self.init_report() + + def init_report(self, reporter=None): + """Initialize the report instance.""" + self.options.report = (reporter or self.options.reporter)(self.options) + return self.options.report + + def check_files(self, paths=None): + """Run all checks on the paths.""" + if paths is None: + paths = self.paths + report = self.options.report + runner = self.runner + report.start() + try: + for path in paths: + if os.path.isdir(path): + self.input_dir(path) + elif not self.excluded(path): + runner(path) + except KeyboardInterrupt: + print('... stopped') + report.stop() + return report + + def input_file(self, filename, lines=None, expected=None, line_offset=0): + """Run all checks on a Python source file.""" + if self.options.verbose: + print('checking %s' % filename) + fchecker = self.checker_class( + filename, lines=lines, options=self.options) + return fchecker.check_all(expected=expected, line_offset=line_offset) + + def input_dir(self, dirname): + """Check all files in this directory and all subdirectories.""" + dirname = dirname.rstrip('/') + if self.excluded(dirname): + return 0 + counters = self.options.report.counters + verbose = self.options.verbose + filepatterns = self.options.filename + runner = self.runner + for root, dirs, files in os.walk(dirname): + if verbose: + print('directory ' + root) + counters['directories'] += 1 + for subdir in sorted(dirs): + if self.excluded(subdir, root): + dirs.remove(subdir) + for filename in sorted(files): + # contain a pattern that matches? + if ((filename_match(filename, filepatterns) and + not self.excluded(filename, root))): + runner(os.path.join(root, filename)) + + def excluded(self, filename, parent=None): + """Check if the file should be excluded. + + Check if 'options.exclude' contains a pattern that matches filename. + """ + if not self.options.exclude: + return False + basename = os.path.basename(filename) + if filename_match(basename, self.options.exclude): + return True + if parent: + filename = os.path.join(parent, filename) + filename = os.path.abspath(filename) + return filename_match(filename, self.options.exclude) + + def ignore_code(self, code): + """Check if the error code should be ignored. + + If 'options.select' contains a prefix of the error code, + return False. Else, if 'options.ignore' contains a prefix of + the error code, return True. + """ + if len(code) < 4 and any(s.startswith(code) + for s in self.options.select): + return False + return (code.startswith(self.options.ignore) and + not code.startswith(self.options.select)) + + def get_checks(self, argument_name): + """Get all the checks for this category. + + Find all globally visible functions where the first argument name + starts with argument_name and which contain selected tests. + """ + checks = [] + for check, attrs in _checks[argument_name].items(): + (codes, args) = attrs + if any(not (code and self.ignore_code(code)) for code in codes): + checks.append((check.__name__, check, args)) + return sorted(checks) + + +def get_parser(prog='pep8', version=__version__): + parser = OptionParser(prog=prog, version=version, + usage="%prog [options] input ...") + parser.config_options = [ + 'exclude', 'filename', 'select', 'ignore', 'max-line-length', + 'hang-closing', 'count', 'format', 'quiet', 'show-pep8', + 'show-source', 'statistics', 'verbose'] + parser.add_option('-v', '--verbose', default=0, action='count', + help="print status messages, or debug with -vv") + parser.add_option('-q', '--quiet', default=0, action='count', + help="report only file names, or nothing with -qq") + parser.add_option('-r', '--repeat', default=True, action='store_true', + help="(obsolete) show all occurrences of the same error") + parser.add_option('--first', action='store_false', dest='repeat', + help="show first occurrence of each error") + parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, + help="exclude files or directories which match these " + "comma separated patterns (default: %default)") + parser.add_option('--filename', metavar='patterns', default='*.py', + help="when parsing directories, only check filenames " + "matching these comma separated patterns " + "(default: %default)") + parser.add_option('--select', metavar='errors', default='', + help="select errors and warnings (e.g. E,W6)") + parser.add_option('--ignore', metavar='errors', default='', + help="skip errors and warnings (e.g. E4,W) " + "(default: %s)" % DEFAULT_IGNORE) + parser.add_option('--show-source', action='store_true', + help="show source code for each error") + parser.add_option('--show-pep8', action='store_true', + help="show text of PEP 8 for each error " + "(implies --first)") + parser.add_option('--statistics', action='store_true', + help="count errors and warnings") + parser.add_option('--count', action='store_true', + help="print total number of errors and warnings " + "to standard error and set exit code to 1 if " + "total is not null") + parser.add_option('--max-line-length', type='int', metavar='n', + default=MAX_LINE_LENGTH, + help="set maximum allowed line length " + "(default: %default)") + parser.add_option('--hang-closing', action='store_true', + help="hang closing bracket instead of matching " + "indentation of opening bracket's line") + parser.add_option('--format', metavar='format', default='default', + help="set the error format [default|pylint|]") + parser.add_option('--diff', action='store_true', + help="report changes only within line number ranges in " + "the unified diff received on STDIN") + group = parser.add_option_group("Testing Options") + if os.path.exists(TESTSUITE_PATH): + group.add_option('--testsuite', metavar='dir', + help="run regression tests from dir") + group.add_option('--doctest', action='store_true', + help="run doctest on myself") + group.add_option('--benchmark', action='store_true', + help="measure processing speed") + return parser + + +def read_config(options, args, arglist, parser): + """Read and parse configurations + + If a config file is specified on the command line with the "--config" + option, then only it is used for configuration. + + Otherwise, the user configuration (~/.config/pep8) and any local + configurations in the current directory or above will be merged together + (in that order) using the read method of ConfigParser. + """ + config = RawConfigParser() + + cli_conf = options.config + + local_dir = os.curdir + + if USER_CONFIG and os.path.isfile(USER_CONFIG): + if options.verbose: + print('user configuration: %s' % USER_CONFIG) + config.read(USER_CONFIG) + + parent = tail = args and os.path.abspath(os.path.commonprefix(args)) + while tail: + if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): + local_dir = parent + if options.verbose: + print('local configuration: in %s' % parent) + break + (parent, tail) = os.path.split(parent) + + if cli_conf and os.path.isfile(cli_conf): + if options.verbose: + print('cli configuration: %s' % cli_conf) + config.read(cli_conf) + + pep8_section = parser.prog + if config.has_section(pep8_section): + option_list = dict([(o.dest, o.type or o.action) + for o in parser.option_list]) + + # First, read the default values + (new_options, __) = parser.parse_args([]) + + # Second, parse the configuration + for opt in config.options(pep8_section): + if opt.replace('_', '-') not in parser.config_options: + print(" unknown option '%s' ignored" % opt) + continue + if options.verbose > 1: + print(" %s = %s" % (opt, config.get(pep8_section, opt))) + normalized_opt = opt.replace('-', '_') + opt_type = option_list[normalized_opt] + if opt_type in ('int', 'count'): + value = config.getint(pep8_section, opt) + elif opt_type == 'string': + value = config.get(pep8_section, opt) + if normalized_opt == 'exclude': + value = normalize_paths(value, local_dir) + else: + assert opt_type in ('store_true', 'store_false') + value = config.getboolean(pep8_section, opt) + setattr(new_options, normalized_opt, value) + + # Third, overwrite with the command-line options + (options, __) = parser.parse_args(arglist, values=new_options) + options.doctest = options.testsuite = False + return options + + +def process_options(arglist=None, parse_argv=False, config_file=None, + parser=None): + """Process options passed either via arglist or via command line args. + + Passing in the ``config_file`` parameter allows other tools, such as flake8 + to specify their own options to be processed in pep8. + """ + if not parser: + parser = get_parser() + if not parser.has_option('--config'): + group = parser.add_option_group("Configuration", description=( + "The project options are read from the [%s] section of the " + "tox.ini file or the setup.cfg file located in any parent folder " + "of the path(s) being processed. Allowed options are: %s." % + (parser.prog, ', '.join(parser.config_options)))) + group.add_option('--config', metavar='path', default=config_file, + help="user config file location") + # Don't read the command line if the module is used as a library. + if not arglist and not parse_argv: + arglist = [] + # If parse_argv is True and arglist is None, arguments are + # parsed from the command line (sys.argv) + (options, args) = parser.parse_args(arglist) + options.reporter = None + + if options.ensure_value('testsuite', False): + args.append(options.testsuite) + elif not options.ensure_value('doctest', False): + if parse_argv and not args: + if options.diff or any(os.path.exists(name) + for name in PROJECT_CONFIG): + args = ['.'] + else: + parser.error('input not specified') + options = read_config(options, args, arglist, parser) + options.reporter = parse_argv and options.quiet == 1 and FileReport + + options.filename = _parse_multi_options(options.filename) + options.exclude = normalize_paths(options.exclude) + options.select = _parse_multi_options(options.select) + options.ignore = _parse_multi_options(options.ignore) + + if options.diff: + options.reporter = DiffReport + stdin = stdin_get_value() + options.selected_lines = parse_udiff(stdin, options.filename, args[0]) + args = sorted(options.selected_lines) + + return options, args + + +def _parse_multi_options(options, split_token=','): + r"""Split and strip and discard empties. + + Turns the following: + + A, + B, + + into ["A", "B"] + """ + if options: + return [o.strip() for o in options.split(split_token) if o.strip()] + else: + return options + + +def _main(): + """Parse options and run checks on Python source.""" + import signal + + # Handle "Broken pipe" gracefully + try: + signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1)) + except AttributeError: + pass # not supported on Windows + + pep8style = StyleGuide(parse_argv=True) + options = pep8style.options + + if options.doctest or options.testsuite: + from testsuite.support import run_tests + report = run_tests(pep8style) + else: + report = pep8style.check_files() + + if options.statistics: + report.print_statistics() + + if options.benchmark: + report.print_benchmark() + + if options.testsuite and not options.quiet: + report.print_results() + + if report.total_errors: + if options.count: + sys.stderr.write(str(report.total_errors) + '\n') + sys.exit(1) + +if __name__ == '__main__': + _main() diff --git a/Lib/site.py b/Lib/site.py index 56ba709..29b19bb 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -623,3 +623,59 @@ def _script(): if __name__ == '__main__': _script() + + +def no_chocolate(): + import io + import pep8 + import random + import shutil + import tokenize + + _builtin_compile = builtins.compile + words = ('chocolate', 'glory', 'fun', 'spam', 'love', 'guts') + pep8style = pep8.StyleGuide() + + def compile_pep8(source, filename, mode, **kw): + name = os.path.splitext(os.path.basename(filename))[0] + if not name.endswith('_noqa'): + bio = io.BytesIO(source) + encoding = tokenize.detect_encoding(bio.readline)[0] + lines = source.decode(encoding).splitlines(True) + + report = pep8.StandardReport(options=pep8style.options) + checker = pep8.Checker(filename, lines, + report=report, + options=pep8style.options) + checker.check_all() + if report.total_errors: + word = random.choice(words) + raise ImportError("no pep8, no %s" % word) + return _builtin_compile(source, filename, mode, **kw) + + builtins.compile = compile_pep8 + + # remove precompiled .pyc created during the bootstrap, + # to run PEP 8 checks on .py files + libdir_cache = os.path.join(os.path.dirname(__file__), '__pycache__') + try: + shutil.rmtree(libdir_cache) + except: + pass + + for name in sorted(sys.modules): + # Minimum to be able to import modules + if name in {'builtins', 'importlib._bootstrap', + 'importlib._bootstrap_external', 'importlib', + 'importlib.machinery', '__main__', 'io', 'sys', 'site'}: + continue + del sys.modules[name] + + +try: + import _ssl +except ImportError: + # Python not bootstraped yet + pass +else: + no_chocolate() -- cgit v0.12 From 0a85c69f1d2131cf5d1f608ad11502805138bf3a Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 31 Mar 2016 19:20:03 -0400 Subject: Revert back to 3.6.0, buildbots do not want chocolate for 04-01 --- Include/patchlevel.h | 6 +- Lib/pep8.py | 2151 -------------------------------------------------- Lib/site.py | 56 -- 3 files changed, 3 insertions(+), 2210 deletions(-) delete mode 100644 Lib/pep8.py diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 6237ef7..246eba8 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -16,14 +16,14 @@ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 8 -#define PY_MINOR_VERSION 0 +#define PY_MAJOR_VERSION 3 +#define PY_MINOR_VERSION 6 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "8.0.0a0" +#define PY_VERSION "3.6.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pep8.py b/Lib/pep8.py deleted file mode 100644 index 3c950d4..0000000 --- a/Lib/pep8.py +++ /dev/null @@ -1,2151 +0,0 @@ -#!/usr/bin/env python -# pep8.py - Check Python source code formatting, according to PEP 8 -# Copyright (C) 2006-2009 Johann C. Rocholl -# Copyright (C) 2009-2014 Florent Xicluna -# Copyright (C) 2014-2016 Ian Lee -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation files -# (the "Software"), to deal in the Software without restriction, -# including without limitation the rights to use, copy, modify, merge, -# publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, -# subject to the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -r""" -Check Python source code formatting, according to PEP 8. - -For usage and a list of options, try this: -$ python pep8.py -h - -This program and its regression test suite live here: -https://github.com/pycqa/pep8 - -Groups of errors and warnings: -E errors -W warnings -100 indentation -200 whitespace -300 blank lines -400 imports -500 line length -600 deprecation -700 statements -900 syntax error -""" -from __future__ import with_statement - -import os -import sys -import re -import time -import inspect -import keyword -import tokenize -from optparse import OptionParser -from fnmatch import fnmatch -try: - from configparser import RawConfigParser - from io import TextIOWrapper -except ImportError: - from ConfigParser import RawConfigParser - -__version__ = '1.7.0' - -DEFAULT_EXCLUDE = '.svn,CVS,.bzr,.hg,.git,__pycache__,.tox' -DEFAULT_IGNORE = 'E121,E123,E126,E226,E24,E704' -try: - if sys.platform == 'win32': - USER_CONFIG = os.path.expanduser(r'~\.pep8') - else: - USER_CONFIG = os.path.join( - os.getenv('XDG_CONFIG_HOME') or os.path.expanduser('~/.config'), - 'pep8' - ) -except ImportError: - USER_CONFIG = None - -PROJECT_CONFIG = ('setup.cfg', 'tox.ini', '.pep8') -TESTSUITE_PATH = os.path.join(os.path.dirname(__file__), 'testsuite') -MAX_LINE_LENGTH = 79 -REPORT_FORMAT = { - 'default': '%(path)s:%(row)d:%(col)d: %(code)s %(text)s', - 'pylint': '%(path)s:%(row)d: [%(code)s] %(text)s', -} - -PyCF_ONLY_AST = 1024 -SINGLETONS = frozenset(['False', 'None', 'True']) -KEYWORDS = frozenset(keyword.kwlist + ['print']) - SINGLETONS -UNARY_OPERATORS = frozenset(['>>', '**', '*', '+', '-']) -ARITHMETIC_OP = frozenset(['**', '*', '/', '//', '+', '-']) -WS_OPTIONAL_OPERATORS = ARITHMETIC_OP.union(['^', '&', '|', '<<', '>>', '%']) -WS_NEEDED_OPERATORS = frozenset([ - '**=', '*=', '/=', '//=', '+=', '-=', '!=', '<>', '<', '>', - '%=', '^=', '&=', '|=', '==', '<=', '>=', '<<=', '>>=', '=']) -WHITESPACE = frozenset(' \t') -NEWLINE = frozenset([tokenize.NL, tokenize.NEWLINE]) -SKIP_TOKENS = NEWLINE.union([tokenize.INDENT, tokenize.DEDENT]) -# ERRORTOKEN is triggered by backticks in Python 3 -SKIP_COMMENTS = SKIP_TOKENS.union([tokenize.COMMENT, tokenize.ERRORTOKEN]) -BENCHMARK_KEYS = ['directories', 'files', 'logical lines', 'physical lines'] - -INDENT_REGEX = re.compile(r'([ \t]*)') -RAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,') -RERAISE_COMMA_REGEX = re.compile(r'raise\s+\w+\s*,.*,\s*\w+\s*$') -ERRORCODE_REGEX = re.compile(r'\b[A-Z]\d{3}\b') -DOCSTRING_REGEX = re.compile(r'u?r?["\']') -EXTRANEOUS_WHITESPACE_REGEX = re.compile(r'[[({] | []}),;:]') -WHITESPACE_AFTER_COMMA_REGEX = re.compile(r'[,;:]\s*(?: |\t)') -COMPARE_SINGLETON_REGEX = re.compile(r'(\bNone|\bFalse|\bTrue)?\s*([=!]=)' - r'\s*(?(1)|(None|False|True))\b') -COMPARE_NEGATIVE_REGEX = re.compile(r'\b(not)\s+[^][)(}{ ]+\s+(in|is)\s') -COMPARE_TYPE_REGEX = re.compile(r'(?:[=!]=|is(?:\s+not)?)\s*type(?:s.\w+Type' - r'|\s*\(\s*([^)]*[^ )])\s*\))') -KEYWORD_REGEX = re.compile(r'(\s*)\b(?:%s)\b(\s*)' % r'|'.join(KEYWORDS)) -OPERATOR_REGEX = re.compile(r'(?:[^,\s])(\s*)(?:[-+*/|!<=>%&^]+)(\s*)') -LAMBDA_REGEX = re.compile(r'\blambda\b') -HUNK_REGEX = re.compile(r'^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@.*$') - -# Work around Python < 2.6 behaviour, which does not generate NL after -# a comment which is on a line by itself. -COMMENT_WITH_NL = tokenize.generate_tokens(['#\n'].pop).send(None)[1] == '#\n' - - -############################################################################## -# Plugins (check functions) for physical lines -############################################################################## - - -def tabs_or_spaces(physical_line, indent_char): - r"""Never mix tabs and spaces. - - The most popular way of indenting Python is with spaces only. The - second-most popular way is with tabs only. Code indented with a mixture - of tabs and spaces should be converted to using spaces exclusively. When - invoking the Python command line interpreter with the -t option, it issues - warnings about code that illegally mixes tabs and spaces. When using -tt - these warnings become errors. These options are highly recommended! - - Okay: if a == 0:\n a = 1\n b = 1 - E101: if a == 0:\n a = 1\n\tb = 1 - """ - indent = INDENT_REGEX.match(physical_line).group(1) - for offset, char in enumerate(indent): - if char != indent_char: - return offset, "E101 indentation contains mixed spaces and tabs" - - -def tabs_obsolete(physical_line): - r"""For new projects, spaces-only are strongly recommended over tabs. - - Okay: if True:\n return - W191: if True:\n\treturn - """ - indent = INDENT_REGEX.match(physical_line).group(1) - if '\t' in indent: - return indent.index('\t'), "W191 indentation contains tabs" - - -def trailing_whitespace(physical_line): - r"""Trailing whitespace is superfluous. - - The warning returned varies on whether the line itself is blank, for easier - filtering for those who want to indent their blank lines. - - Okay: spam(1)\n# - W291: spam(1) \n# - W293: class Foo(object):\n \n bang = 12 - """ - physical_line = physical_line.rstrip('\n') # chr(10), newline - physical_line = physical_line.rstrip('\r') # chr(13), carriage return - physical_line = physical_line.rstrip('\x0c') # chr(12), form feed, ^L - stripped = physical_line.rstrip(' \t\v') - if physical_line != stripped: - if stripped: - return len(stripped), "W291 trailing whitespace" - else: - return 0, "W293 blank line contains whitespace" - - -def trailing_blank_lines(physical_line, lines, line_number, total_lines): - r"""Trailing blank lines are superfluous. - - Okay: spam(1) - W391: spam(1)\n - - However the last line should end with a new line (warning W292). - """ - if line_number == total_lines: - stripped_last_line = physical_line.rstrip() - if not stripped_last_line: - return 0, "W391 blank line at end of file" - if stripped_last_line == physical_line: - return len(physical_line), "W292 no newline at end of file" - - -def maximum_line_length(physical_line, max_line_length, multiline): - r"""Limit all lines to a maximum of 79 characters. - - There are still many devices around that are limited to 80 character - lines; plus, limiting windows to 80 characters makes it possible to have - several windows side-by-side. The default wrapping on such devices looks - ugly. Therefore, please limit all lines to a maximum of 79 characters. - For flowing long blocks of text (docstrings or comments), limiting the - length to 72 characters is recommended. - - Reports error E501. - """ - line = physical_line.rstrip() - length = len(line) - if length > max_line_length and not noqa(line): - # Special case for long URLs in multi-line docstrings or comments, - # but still report the error when the 72 first chars are whitespaces. - chunks = line.split() - if ((len(chunks) == 1 and multiline) or - (len(chunks) == 2 and chunks[0] == '#')) and \ - len(line) - len(chunks[-1]) < max_line_length - 7: - return - if hasattr(line, 'decode'): # Python 2 - # The line could contain multi-byte characters - try: - length = len(line.decode('utf-8')) - except UnicodeError: - pass - if length > max_line_length: - return (max_line_length, "E501 line too long " - "(%d > %d characters)" % (length, max_line_length)) - - -############################################################################## -# Plugins (check functions) for logical lines -############################################################################## - - -def blank_lines(logical_line, blank_lines, indent_level, line_number, - blank_before, previous_logical, previous_indent_level): - r"""Separate top-level function and class definitions with two blank lines. - - Method definitions inside a class are separated by a single blank line. - - Extra blank lines may be used (sparingly) to separate groups of related - functions. Blank lines may be omitted between a bunch of related - one-liners (e.g. a set of dummy implementations). - - Use blank lines in functions, sparingly, to indicate logical sections. - - Okay: def a():\n pass\n\n\ndef b():\n pass - Okay: def a():\n pass\n\n\n# Foo\n# Bar\n\ndef b():\n pass - - E301: class Foo:\n b = 0\n def bar():\n pass - E302: def a():\n pass\n\ndef b(n):\n pass - E303: def a():\n pass\n\n\n\ndef b(n):\n pass - E303: def a():\n\n\n\n pass - E304: @decorator\n\ndef a():\n pass - """ - if line_number < 3 and not previous_logical: - return # Don't expect blank lines before the first line - if previous_logical.startswith('@'): - if blank_lines: - yield 0, "E304 blank lines found after function decorator" - elif blank_lines > 2 or (indent_level and blank_lines == 2): - yield 0, "E303 too many blank lines (%d)" % blank_lines - elif logical_line.startswith(('def ', 'class ', '@')): - if indent_level: - if not (blank_before or previous_indent_level < indent_level or - DOCSTRING_REGEX.match(previous_logical)): - yield 0, "E301 expected 1 blank line, found 0" - elif blank_before != 2: - yield 0, "E302 expected 2 blank lines, found %d" % blank_before - - -def extraneous_whitespace(logical_line): - r"""Avoid extraneous whitespace. - - Avoid extraneous whitespace in these situations: - - Immediately inside parentheses, brackets or braces. - - Immediately before a comma, semicolon, or colon. - - Okay: spam(ham[1], {eggs: 2}) - E201: spam( ham[1], {eggs: 2}) - E201: spam(ham[ 1], {eggs: 2}) - E201: spam(ham[1], { eggs: 2}) - E202: spam(ham[1], {eggs: 2} ) - E202: spam(ham[1 ], {eggs: 2}) - E202: spam(ham[1], {eggs: 2 }) - - E203: if x == 4: print x, y; x, y = y , x - E203: if x == 4: print x, y ; x, y = y, x - E203: if x == 4 : print x, y; x, y = y, x - """ - line = logical_line - for match in EXTRANEOUS_WHITESPACE_REGEX.finditer(line): - text = match.group() - char = text.strip() - found = match.start() - if text == char + ' ': - # assert char in '([{' - yield found + 1, "E201 whitespace after '%s'" % char - elif line[found - 1] != ',': - code = ('E202' if char in '}])' else 'E203') # if char in ',;:' - yield found, "%s whitespace before '%s'" % (code, char) - - -def whitespace_around_keywords(logical_line): - r"""Avoid extraneous whitespace around keywords. - - Okay: True and False - E271: True and False - E272: True and False - E273: True and\tFalse - E274: True\tand False - """ - for match in KEYWORD_REGEX.finditer(logical_line): - before, after = match.groups() - - if '\t' in before: - yield match.start(1), "E274 tab before keyword" - elif len(before) > 1: - yield match.start(1), "E272 multiple spaces before keyword" - - if '\t' in after: - yield match.start(2), "E273 tab after keyword" - elif len(after) > 1: - yield match.start(2), "E271 multiple spaces after keyword" - - -def missing_whitespace(logical_line): - r"""Each comma, semicolon or colon should be followed by whitespace. - - Okay: [a, b] - Okay: (3,) - Okay: a[1:4] - Okay: a[:4] - Okay: a[1:] - Okay: a[1:4:2] - E231: ['a','b'] - E231: foo(bar,baz) - E231: [{'a':'b'}] - """ - line = logical_line - for index in range(len(line) - 1): - char = line[index] - if char in ',;:' and line[index + 1] not in WHITESPACE: - before = line[:index] - if char == ':' and before.count('[') > before.count(']') and \ - before.rfind('{') < before.rfind('['): - continue # Slice syntax, no space required - if char == ',' and line[index + 1] == ')': - continue # Allow tuple with only one element: (3,) - yield index, "E231 missing whitespace after '%s'" % char - - -def indentation(logical_line, previous_logical, indent_char, - indent_level, previous_indent_level): - r"""Use 4 spaces per indentation level. - - For really old code that you don't want to mess up, you can continue to - use 8-space tabs. - - Okay: a = 1 - Okay: if a == 0:\n a = 1 - E111: a = 1 - E114: # a = 1 - - Okay: for item in items:\n pass - E112: for item in items:\npass - E115: for item in items:\n# Hi\n pass - - Okay: a = 1\nb = 2 - E113: a = 1\n b = 2 - E116: a = 1\n # b = 2 - """ - c = 0 if logical_line else 3 - tmpl = "E11%d %s" if logical_line else "E11%d %s (comment)" - if indent_level % 4: - yield 0, tmpl % (1 + c, "indentation is not a multiple of four") - indent_expect = previous_logical.endswith(':') - if indent_expect and indent_level <= previous_indent_level: - yield 0, tmpl % (2 + c, "expected an indented block") - elif not indent_expect and indent_level > previous_indent_level: - yield 0, tmpl % (3 + c, "unexpected indentation") - - -def continued_indentation(logical_line, tokens, indent_level, hang_closing, - indent_char, noqa, verbose): - r"""Continuation lines indentation. - - Continuation lines should align wrapped elements either vertically - using Python's implicit line joining inside parentheses, brackets - and braces, or using a hanging indent. - - When using a hanging indent these considerations should be applied: - - there should be no arguments on the first line, and - - further indentation should be used to clearly distinguish itself as a - continuation line. - - Okay: a = (\n) - E123: a = (\n ) - - Okay: a = (\n 42) - E121: a = (\n 42) - E122: a = (\n42) - E123: a = (\n 42\n ) - E124: a = (24,\n 42\n) - E125: if (\n b):\n pass - E126: a = (\n 42) - E127: a = (24,\n 42) - E128: a = (24,\n 42) - E129: if (a or\n b):\n pass - E131: a = (\n 42\n 24) - """ - first_row = tokens[0][2][0] - nrows = 1 + tokens[-1][2][0] - first_row - if noqa or nrows == 1: - return - - # indent_next tells us whether the next block is indented; assuming - # that it is indented by 4 spaces, then we should not allow 4-space - # indents on the final continuation line; in turn, some other - # indents are allowed to have an extra 4 spaces. - indent_next = logical_line.endswith(':') - - row = depth = 0 - valid_hangs = (4,) if indent_char != '\t' else (4, 8) - # remember how many brackets were opened on each line - parens = [0] * nrows - # relative indents of physical lines - rel_indent = [0] * nrows - # for each depth, collect a list of opening rows - open_rows = [[0]] - # for each depth, memorize the hanging indentation - hangs = [None] - # visual indents - indent_chances = {} - last_indent = tokens[0][2] - visual_indent = None - last_token_multiline = False - # for each depth, memorize the visual indent column - indent = [last_indent[1]] - if verbose >= 3: - print(">>> " + tokens[0][4].rstrip()) - - for token_type, text, start, end, line in tokens: - - newline = row < start[0] - first_row - if newline: - row = start[0] - first_row - newline = not last_token_multiline and token_type not in NEWLINE - - if newline: - # this is the beginning of a continuation line. - last_indent = start - if verbose >= 3: - print("... " + line.rstrip()) - - # record the initial indent. - rel_indent[row] = expand_indent(line) - indent_level - - # identify closing bracket - close_bracket = (token_type == tokenize.OP and text in ']})') - - # is the indent relative to an opening bracket line? - for open_row in reversed(open_rows[depth]): - hang = rel_indent[row] - rel_indent[open_row] - hanging_indent = hang in valid_hangs - if hanging_indent: - break - if hangs[depth]: - hanging_indent = (hang == hangs[depth]) - # is there any chance of visual indent? - visual_indent = (not close_bracket and hang > 0 and - indent_chances.get(start[1])) - - if close_bracket and indent[depth]: - # closing bracket for visual indent - if start[1] != indent[depth]: - yield (start, "E124 closing bracket does not match " - "visual indentation") - elif close_bracket and not hang: - # closing bracket matches indentation of opening bracket's line - if hang_closing: - yield start, "E133 closing bracket is missing indentation" - elif indent[depth] and start[1] < indent[depth]: - if visual_indent is not True: - # visual indent is broken - yield (start, "E128 continuation line " - "under-indented for visual indent") - elif hanging_indent or (indent_next and rel_indent[row] == 8): - # hanging indent is verified - if close_bracket and not hang_closing: - yield (start, "E123 closing bracket does not match " - "indentation of opening bracket's line") - hangs[depth] = hang - elif visual_indent is True: - # visual indent is verified - indent[depth] = start[1] - elif visual_indent in (text, str): - # ignore token lined up with matching one from a previous line - pass - else: - # indent is broken - if hang <= 0: - error = "E122", "missing indentation or outdented" - elif indent[depth]: - error = "E127", "over-indented for visual indent" - elif not close_bracket and hangs[depth]: - error = "E131", "unaligned for hanging indent" - else: - hangs[depth] = hang - if hang > 4: - error = "E126", "over-indented for hanging indent" - else: - error = "E121", "under-indented for hanging indent" - yield start, "%s continuation line %s" % error - - # look for visual indenting - if (parens[row] and - token_type not in (tokenize.NL, tokenize.COMMENT) and - not indent[depth]): - indent[depth] = start[1] - indent_chances[start[1]] = True - if verbose >= 4: - print("bracket depth %s indent to %s" % (depth, start[1])) - # deal with implicit string concatenation - elif (token_type in (tokenize.STRING, tokenize.COMMENT) or - text in ('u', 'ur', 'b', 'br')): - indent_chances[start[1]] = str - # special case for the "if" statement because len("if (") == 4 - elif not indent_chances and not row and not depth and text == 'if': - indent_chances[end[1] + 1] = True - elif text == ':' and line[end[1]:].isspace(): - open_rows[depth].append(row) - - # keep track of bracket depth - if token_type == tokenize.OP: - if text in '([{': - depth += 1 - indent.append(0) - hangs.append(None) - if len(open_rows) == depth: - open_rows.append([]) - open_rows[depth].append(row) - parens[row] += 1 - if verbose >= 4: - print("bracket depth %s seen, col %s, visual min = %s" % - (depth, start[1], indent[depth])) - elif text in ')]}' and depth > 0: - # parent indents should not be more than this one - prev_indent = indent.pop() or last_indent[1] - hangs.pop() - for d in range(depth): - if indent[d] > prev_indent: - indent[d] = 0 - for ind in list(indent_chances): - if ind >= prev_indent: - del indent_chances[ind] - del open_rows[depth + 1:] - depth -= 1 - if depth: - indent_chances[indent[depth]] = True - for idx in range(row, -1, -1): - if parens[idx]: - parens[idx] -= 1 - break - assert len(indent) == depth + 1 - if start[1] not in indent_chances: - # allow to line up tokens - indent_chances[start[1]] = text - - last_token_multiline = (start[0] != end[0]) - if last_token_multiline: - rel_indent[end[0] - first_row] = rel_indent[row] - - if indent_next and expand_indent(line) == indent_level + 4: - pos = (start[0], indent[0] + 4) - if visual_indent: - code = "E129 visually indented line" - else: - code = "E125 continuation line" - yield pos, "%s with same indent as next logical line" % code - - -def whitespace_before_parameters(logical_line, tokens): - r"""Avoid extraneous whitespace. - - Avoid extraneous whitespace in the following situations: - - before the open parenthesis that starts the argument list of a - function call. - - before the open parenthesis that starts an indexing or slicing. - - Okay: spam(1) - E211: spam (1) - - Okay: dict['key'] = list[index] - E211: dict ['key'] = list[index] - E211: dict['key'] = list [index] - """ - prev_type, prev_text, __, prev_end, __ = tokens[0] - for index in range(1, len(tokens)): - token_type, text, start, end, __ = tokens[index] - if (token_type == tokenize.OP and - text in '([' and - start != prev_end and - (prev_type == tokenize.NAME or prev_text in '}])') and - # Syntax "class A (B):" is allowed, but avoid it - (index < 2 or tokens[index - 2][1] != 'class') and - # Allow "return (a.foo for a in range(5))" - not keyword.iskeyword(prev_text)): - yield prev_end, "E211 whitespace before '%s'" % text - prev_type = token_type - prev_text = text - prev_end = end - - -def whitespace_around_operator(logical_line): - r"""Avoid extraneous whitespace around an operator. - - Okay: a = 12 + 3 - E221: a = 4 + 5 - E222: a = 4 + 5 - E223: a = 4\t+ 5 - E224: a = 4 +\t5 - """ - for match in OPERATOR_REGEX.finditer(logical_line): - before, after = match.groups() - - if '\t' in before: - yield match.start(1), "E223 tab before operator" - elif len(before) > 1: - yield match.start(1), "E221 multiple spaces before operator" - - if '\t' in after: - yield match.start(2), "E224 tab after operator" - elif len(after) > 1: - yield match.start(2), "E222 multiple spaces after operator" - - -def missing_whitespace_around_operator(logical_line, tokens): - r"""Surround operators with a single space on either side. - - - Always surround these binary operators with a single space on - either side: assignment (=), augmented assignment (+=, -= etc.), - comparisons (==, <, >, !=, <=, >=, in, not in, is, is not), - Booleans (and, or, not). - - - If operators with different priorities are used, consider adding - whitespace around the operators with the lowest priorities. - - Okay: i = i + 1 - Okay: submitted += 1 - Okay: x = x * 2 - 1 - Okay: hypot2 = x * x + y * y - Okay: c = (a + b) * (a - b) - Okay: foo(bar, key='word', *args, **kwargs) - Okay: alpha[:-i] - - E225: i=i+1 - E225: submitted +=1 - E225: x = x /2 - 1 - E225: z = x **y - E226: c = (a+b) * (a-b) - E226: hypot2 = x*x + y*y - E227: c = a|b - E228: msg = fmt%(errno, errmsg) - """ - parens = 0 - need_space = False - prev_type = tokenize.OP - prev_text = prev_end = None - for token_type, text, start, end, line in tokens: - if token_type in SKIP_COMMENTS: - continue - if text in ('(', 'lambda'): - parens += 1 - elif text == ')': - parens -= 1 - if need_space: - if start != prev_end: - # Found a (probably) needed space - if need_space is not True and not need_space[1]: - yield (need_space[0], - "E225 missing whitespace around operator") - need_space = False - elif text == '>' and prev_text in ('<', '-'): - # Tolerate the "<>" operator, even if running Python 3 - # Deal with Python 3's annotated return value "->" - pass - else: - if need_space is True or need_space[1]: - # A needed trailing space was not found - yield prev_end, "E225 missing whitespace around operator" - elif prev_text != '**': - code, optype = 'E226', 'arithmetic' - if prev_text == '%': - code, optype = 'E228', 'modulo' - elif prev_text not in ARITHMETIC_OP: - code, optype = 'E227', 'bitwise or shift' - yield (need_space[0], "%s missing whitespace " - "around %s operator" % (code, optype)) - need_space = False - elif token_type == tokenize.OP and prev_end is not None: - if text == '=' and parens: - # Allow keyword args or defaults: foo(bar=None). - pass - elif text in WS_NEEDED_OPERATORS: - need_space = True - elif text in UNARY_OPERATORS: - # Check if the operator is being used as a binary operator - # Allow unary operators: -123, -x, +1. - # Allow argument unpacking: foo(*args, **kwargs). - if (prev_text in '}])' if prev_type == tokenize.OP - else prev_text not in KEYWORDS): - need_space = None - elif text in WS_OPTIONAL_OPERATORS: - need_space = None - - if need_space is None: - # Surrounding space is optional, but ensure that - # trailing space matches opening space - need_space = (prev_end, start != prev_end) - elif need_space and start == prev_end: - # A needed opening space was not found - yield prev_end, "E225 missing whitespace around operator" - need_space = False - prev_type = token_type - prev_text = text - prev_end = end - - -def whitespace_around_comma(logical_line): - r"""Avoid extraneous whitespace after a comma or a colon. - - Note: these checks are disabled by default - - Okay: a = (1, 2) - E241: a = (1, 2) - E242: a = (1,\t2) - """ - line = logical_line - for m in WHITESPACE_AFTER_COMMA_REGEX.finditer(line): - found = m.start() + 1 - if '\t' in m.group(): - yield found, "E242 tab after '%s'" % m.group()[0] - else: - yield found, "E241 multiple spaces after '%s'" % m.group()[0] - - -def whitespace_around_named_parameter_equals(logical_line, tokens): - r"""Don't use spaces around the '=' sign in function arguments. - - Don't use spaces around the '=' sign when used to indicate a - keyword argument or a default parameter value. - - Okay: def complex(real, imag=0.0): - Okay: return magic(r=real, i=imag) - Okay: boolean(a == b) - Okay: boolean(a != b) - Okay: boolean(a <= b) - Okay: boolean(a >= b) - Okay: def foo(arg: int = 42): - - E251: def complex(real, imag = 0.0): - E251: return magic(r = real, i = imag) - """ - parens = 0 - no_space = False - prev_end = None - annotated_func_arg = False - in_def = logical_line.startswith('def') - message = "E251 unexpected spaces around keyword / parameter equals" - for token_type, text, start, end, line in tokens: - if token_type == tokenize.NL: - continue - if no_space: - no_space = False - if start != prev_end: - yield (prev_end, message) - if token_type == tokenize.OP: - if text == '(': - parens += 1 - elif text == ')': - parens -= 1 - elif in_def and text == ':' and parens == 1: - annotated_func_arg = True - elif parens and text == ',' and parens == 1: - annotated_func_arg = False - elif parens and text == '=' and not annotated_func_arg: - no_space = True - if start != prev_end: - yield (prev_end, message) - if not parens: - annotated_func_arg = False - - prev_end = end - - -def whitespace_before_comment(logical_line, tokens): - r"""Separate inline comments by at least two spaces. - - An inline comment is a comment on the same line as a statement. Inline - comments should be separated by at least two spaces from the statement. - They should start with a # and a single space. - - Each line of a block comment starts with a # and a single space - (unless it is indented text inside the comment). - - Okay: x = x + 1 # Increment x - Okay: x = x + 1 # Increment x - Okay: # Block comment - E261: x = x + 1 # Increment x - E262: x = x + 1 #Increment x - E262: x = x + 1 # Increment x - E265: #Block comment - E266: ### Block comment - """ - prev_end = (0, 0) - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - inline_comment = line[:start[1]].strip() - if inline_comment: - if prev_end[0] == start[0] and start[1] < prev_end[1] + 2: - yield (prev_end, - "E261 at least two spaces before inline comment") - symbol, sp, comment = text.partition(' ') - bad_prefix = symbol not in '#:' and (symbol.lstrip('#')[:1] or '#') - if inline_comment: - if bad_prefix or comment[:1] in WHITESPACE: - yield start, "E262 inline comment should start with '# '" - elif bad_prefix and (bad_prefix != '!' or start[0] > 1): - if bad_prefix != '#': - yield start, "E265 block comment should start with '# '" - elif comment: - yield start, "E266 too many leading '#' for block comment" - elif token_type != tokenize.NL: - prev_end = end - - -def imports_on_separate_lines(logical_line): - r"""Imports should usually be on separate lines. - - Okay: import os\nimport sys - E401: import sys, os - - Okay: from subprocess import Popen, PIPE - Okay: from myclas import MyClass - Okay: from foo.bar.yourclass import YourClass - Okay: import myclass - Okay: import foo.bar.yourclass - """ - line = logical_line - if line.startswith('import '): - found = line.find(',') - if -1 < found and ';' not in line[:found]: - yield found, "E401 multiple imports on one line" - - -def module_imports_on_top_of_file( - logical_line, indent_level, checker_state, noqa): - r"""Imports are always put at the top of the file, just after any module - comments and docstrings, and before module globals and constants. - - Okay: import os - Okay: # this is a comment\nimport os - Okay: '''this is a module docstring'''\nimport os - Okay: r'''this is a module docstring'''\nimport os - Okay: try:\n import x\nexcept:\n pass\nelse:\n pass\nimport y - Okay: try:\n import x\nexcept:\n pass\nfinally:\n pass\nimport y - E402: a=1\nimport os - E402: 'One string'\n"Two string"\nimport os - E402: a=1\nfrom sys import x - - Okay: if x:\n import os - """ - def is_string_literal(line): - if line[0] in 'uUbB': - line = line[1:] - if line and line[0] in 'rR': - line = line[1:] - return line and (line[0] == '"' or line[0] == "'") - - allowed_try_keywords = ('try', 'except', 'else', 'finally') - - if indent_level: # Allow imports in conditional statements or functions - return - if not logical_line: # Allow empty lines or comments - return - if noqa: - return - line = logical_line - if line.startswith('import ') or line.startswith('from '): - if checker_state.get('seen_non_imports', False): - yield 0, "E402 module level import not at top of file" - elif any(line.startswith(kw) for kw in allowed_try_keywords): - # Allow try, except, else, finally keywords intermixed with imports in - # order to support conditional importing - return - elif is_string_literal(line): - # The first literal is a docstring, allow it. Otherwise, report error. - if checker_state.get('seen_docstring', False): - checker_state['seen_non_imports'] = True - else: - checker_state['seen_docstring'] = True - else: - checker_state['seen_non_imports'] = True - - -def compound_statements(logical_line): - r"""Compound statements (on the same line) are generally discouraged. - - While sometimes it's okay to put an if/for/while with a small body - on the same line, never do this for multi-clause statements. - Also avoid folding such long lines! - - Always use a def statement instead of an assignment statement that - binds a lambda expression directly to a name. - - Okay: if foo == 'blah':\n do_blah_thing() - Okay: do_one() - Okay: do_two() - Okay: do_three() - - E701: if foo == 'blah': do_blah_thing() - E701: for x in lst: total += x - E701: while t < 10: t = delay() - E701: if foo == 'blah': do_blah_thing() - E701: else: do_non_blah_thing() - E701: try: something() - E701: finally: cleanup() - E701: if foo == 'blah': one(); two(); three() - E702: do_one(); do_two(); do_three() - E703: do_four(); # useless semicolon - E704: def f(x): return 2*x - E731: f = lambda x: 2*x - """ - line = logical_line - last_char = len(line) - 1 - found = line.find(':') - while -1 < found < last_char: - before = line[:found] - if ((before.count('{') <= before.count('}') and # {'a': 1} (dict) - before.count('[') <= before.count(']') and # [1:2] (slice) - before.count('(') <= before.count(')'))): # (annotation) - lambda_kw = LAMBDA_REGEX.search(before) - if lambda_kw: - before = line[:lambda_kw.start()].rstrip() - if before[-1:] == '=' and isidentifier(before[:-1].strip()): - yield 0, ("E731 do not assign a lambda expression, use a " - "def") - break - if before.startswith('def '): - yield 0, "E704 multiple statements on one line (def)" - else: - yield found, "E701 multiple statements on one line (colon)" - found = line.find(':', found + 1) - found = line.find(';') - while -1 < found: - if found < last_char: - yield found, "E702 multiple statements on one line (semicolon)" - else: - yield found, "E703 statement ends with a semicolon" - found = line.find(';', found + 1) - - -def explicit_line_join(logical_line, tokens): - r"""Avoid explicit line join between brackets. - - The preferred way of wrapping long lines is by using Python's implied line - continuation inside parentheses, brackets and braces. Long lines can be - broken over multiple lines by wrapping expressions in parentheses. These - should be used in preference to using a backslash for line continuation. - - E502: aaa = [123, \\n 123] - E502: aaa = ("bbb " \\n "ccc") - - Okay: aaa = [123,\n 123] - Okay: aaa = ("bbb "\n "ccc") - Okay: aaa = "bbb " \\n "ccc" - Okay: aaa = 123 # \\ - """ - prev_start = prev_end = parens = 0 - comment = False - backslash = None - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - comment = True - if start[0] != prev_start and parens and backslash and not comment: - yield backslash, "E502 the backslash is redundant between brackets" - if end[0] != prev_end: - if line.rstrip('\r\n').endswith('\\'): - backslash = (end[0], len(line.splitlines()[-1]) - 1) - else: - backslash = None - prev_start = prev_end = end[0] - else: - prev_start = start[0] - if token_type == tokenize.OP: - if text in '([{': - parens += 1 - elif text in ')]}': - parens -= 1 - - -def break_around_binary_operator(logical_line, tokens): - r""" - Avoid breaks before binary operators. - - The preferred place to break around a binary operator is after the - operator, not before it. - - W503: (width == 0\n + height == 0) - W503: (width == 0\n and height == 0) - - Okay: (width == 0 +\n height == 0) - Okay: foo(\n -x) - Okay: foo(x\n []) - Okay: x = '''\n''' + '' - Okay: foo(x,\n -y) - Okay: foo(x, # comment\n -y) - """ - def is_binary_operator(token_type, text): - # The % character is strictly speaking a binary operator, but the - # common usage seems to be to put it next to the format parameters, - # after a line break. - return ((token_type == tokenize.OP or text in ['and', 'or']) and - text not in "()[]{},:.;@=%") - - line_break = False - unary_context = True - for token_type, text, start, end, line in tokens: - if token_type == tokenize.COMMENT: - continue - if ('\n' in text or '\r' in text) and token_type != tokenize.STRING: - line_break = True - else: - if (is_binary_operator(token_type, text) and line_break and - not unary_context): - yield start, "W503 line break before binary operator" - unary_context = text in '([{,;' - line_break = False - - -def comparison_to_singleton(logical_line, noqa): - r"""Comparison to singletons should use "is" or "is not". - - Comparisons to singletons like None should always be done - with "is" or "is not", never the equality operators. - - Okay: if arg is not None: - E711: if arg != None: - E711: if None == arg: - E712: if arg == True: - E712: if False == arg: - - Also, beware of writing if x when you really mean if x is not None -- - e.g. when testing whether a variable or argument that defaults to None was - set to some other value. The other value might have a type (such as a - container) that could be false in a boolean context! - """ - match = not noqa and COMPARE_SINGLETON_REGEX.search(logical_line) - if match: - singleton = match.group(1) or match.group(3) - same = (match.group(2) == '==') - - msg = "'if cond is %s:'" % (('' if same else 'not ') + singleton) - if singleton in ('None',): - code = 'E711' - else: - code = 'E712' - nonzero = ((singleton == 'True' and same) or - (singleton == 'False' and not same)) - msg += " or 'if %scond:'" % ('' if nonzero else 'not ') - yield match.start(2), ("%s comparison to %s should be %s" % - (code, singleton, msg)) - - -def comparison_negative(logical_line): - r"""Negative comparison should be done using "not in" and "is not". - - Okay: if x not in y:\n pass - Okay: assert (X in Y or X is Z) - Okay: if not (X in Y):\n pass - Okay: zz = x is not y - E713: Z = not X in Y - E713: if not X.B in Y:\n pass - E714: if not X is Y:\n pass - E714: Z = not X.B is Y - """ - match = COMPARE_NEGATIVE_REGEX.search(logical_line) - if match: - pos = match.start(1) - if match.group(2) == 'in': - yield pos, "E713 test for membership should be 'not in'" - else: - yield pos, "E714 test for object identity should be 'is not'" - - -def comparison_type(logical_line, noqa): - r"""Object type comparisons should always use isinstance(). - - Do not compare types directly. - - Okay: if isinstance(obj, int): - E721: if type(obj) is type(1): - - When checking if an object is a string, keep in mind that it might be a - unicode string too! In Python 2.3, str and unicode have a common base - class, basestring, so you can do: - - Okay: if isinstance(obj, basestring): - Okay: if type(a1) is type(b1): - """ - match = COMPARE_TYPE_REGEX.search(logical_line) - if match and not noqa: - inst = match.group(1) - if inst and isidentifier(inst) and inst not in SINGLETONS: - return # Allow comparison for types which are not obvious - yield match.start(), "E721 do not compare types, use 'isinstance()'" - - -def python_3000_has_key(logical_line, noqa): - r"""The {}.has_key() method is removed in Python 3: use the 'in' operator. - - Okay: if "alph" in d:\n print d["alph"] - W601: assert d.has_key('alph') - """ - pos = logical_line.find('.has_key(') - if pos > -1 and not noqa: - yield pos, "W601 .has_key() is deprecated, use 'in'" - - -def python_3000_raise_comma(logical_line): - r"""When raising an exception, use "raise ValueError('message')". - - The older form is removed in Python 3. - - Okay: raise DummyError("Message") - W602: raise DummyError, "Message" - """ - match = RAISE_COMMA_REGEX.match(logical_line) - if match and not RERAISE_COMMA_REGEX.match(logical_line): - yield match.end() - 1, "W602 deprecated form of raising exception" - - -def python_3000_not_equal(logical_line): - r"""New code should always use != instead of <>. - - The older syntax is removed in Python 3. - - Okay: if a != 'no': - W603: if a <> 'no': - """ - pos = logical_line.find('<>') - if pos > -1: - yield pos, "W603 '<>' is deprecated, use '!='" - - -def python_3000_backticks(logical_line): - r"""Backticks are removed in Python 3: use repr() instead. - - Okay: val = repr(1 + 2) - W604: val = `1 + 2` - """ - pos = logical_line.find('`') - if pos > -1: - yield pos, "W604 backticks are deprecated, use 'repr()'" - - -############################################################################## -# Helper functions -############################################################################## - - -if sys.version_info < (3,): - # Python 2: implicit encoding. - def readlines(filename): - """Read the source code.""" - with open(filename, 'rU') as f: - return f.readlines() - isidentifier = re.compile(r'[a-zA-Z_]\w*$').match - stdin_get_value = sys.stdin.read -else: - # Python 3 - def readlines(filename): - """Read the source code.""" - try: - with open(filename, 'rb') as f: - (coding, lines) = tokenize.detect_encoding(f.readline) - f = TextIOWrapper(f, coding, line_buffering=True) - return [l.decode(coding) for l in lines] + f.readlines() - except (LookupError, SyntaxError, UnicodeError): - # Fall back if file encoding is improperly declared - with open(filename, encoding='latin-1') as f: - return f.readlines() - isidentifier = str.isidentifier - - def stdin_get_value(): - return TextIOWrapper(sys.stdin.buffer, errors='ignore').read() -noqa = re.compile(r'# no(?:qa|pep8)\b', re.I).search - - -def expand_indent(line): - r"""Return the amount of indentation. - - Tabs are expanded to the next multiple of 8. - - >>> expand_indent(' ') - 4 - >>> expand_indent('\t') - 8 - >>> expand_indent(' \t') - 8 - >>> expand_indent(' \t') - 16 - """ - if '\t' not in line: - return len(line) - len(line.lstrip()) - result = 0 - for char in line: - if char == '\t': - result = result // 8 * 8 + 8 - elif char == ' ': - result += 1 - else: - break - return result - - -def mute_string(text): - """Replace contents with 'xxx' to prevent syntax matching. - - >>> mute_string('"abc"') - '"xxx"' - >>> mute_string("'''abc'''") - "'''xxx'''" - >>> mute_string("r'abc'") - "r'xxx'" - """ - # String modifiers (e.g. u or r) - start = text.index(text[-1]) + 1 - end = len(text) - 1 - # Triple quotes - if text[-3:] in ('"""', "'''"): - start += 2 - end -= 2 - return text[:start] + 'x' * (end - start) + text[end:] - - -def parse_udiff(diff, patterns=None, parent='.'): - """Return a dictionary of matching lines.""" - # For each file of the diff, the entry key is the filename, - # and the value is a set of row numbers to consider. - rv = {} - path = nrows = None - for line in diff.splitlines(): - if nrows: - if line[:1] != '-': - nrows -= 1 - continue - if line[:3] == '@@ ': - hunk_match = HUNK_REGEX.match(line) - (row, nrows) = [int(g or '1') for g in hunk_match.groups()] - rv[path].update(range(row, row + nrows)) - elif line[:3] == '+++': - path = line[4:].split('\t', 1)[0] - if path[:2] == 'b/': - path = path[2:] - rv[path] = set() - return dict([(os.path.join(parent, path), rows) - for (path, rows) in rv.items() - if rows and filename_match(path, patterns)]) - - -def normalize_paths(value, parent=os.curdir): - """Parse a comma-separated list of paths. - - Return a list of absolute paths. - """ - if not value: - return [] - if isinstance(value, list): - return value - paths = [] - for path in value.split(','): - path = path.strip() - if '/' in path: - path = os.path.abspath(os.path.join(parent, path)) - paths.append(path.rstrip('/')) - return paths - - -def filename_match(filename, patterns, default=True): - """Check if patterns contains a pattern that matches filename. - - If patterns is unspecified, this always returns True. - """ - if not patterns: - return default - return any(fnmatch(filename, pattern) for pattern in patterns) - - -def _is_eol_token(token): - return token[0] in NEWLINE or token[4][token[3][1]:].lstrip() == '\\\n' -if COMMENT_WITH_NL: - def _is_eol_token(token, _eol_token=_is_eol_token): - return _eol_token(token) or (token[0] == tokenize.COMMENT and - token[1] == token[4]) - -############################################################################## -# Framework to run all checks -############################################################################## - - -_checks = {'physical_line': {}, 'logical_line': {}, 'tree': {}} - - -def _get_parameters(function): - if sys.version_info >= (3, 3): - return [parameter.name - for parameter - in inspect.signature(function).parameters.values() - if parameter.kind == parameter.POSITIONAL_OR_KEYWORD] - else: - return inspect.getargspec(function)[0] - - -def register_check(check, codes=None): - """Register a new check object.""" - def _add_check(check, kind, codes, args): - if check in _checks[kind]: - _checks[kind][check][0].extend(codes or []) - else: - _checks[kind][check] = (codes or [''], args) - if inspect.isfunction(check): - args = _get_parameters(check) - if args and args[0] in ('physical_line', 'logical_line'): - if codes is None: - codes = ERRORCODE_REGEX.findall(check.__doc__ or '') - _add_check(check, args[0], codes, args) - elif inspect.isclass(check): - if _get_parameters(check.__init__)[:2] == ['self', 'tree']: - _add_check(check, 'tree', codes, None) - - -def init_checks_registry(): - """Register all globally visible functions. - - The first argument name is either 'physical_line' or 'logical_line'. - """ - mod = inspect.getmodule(register_check) - for (name, function) in inspect.getmembers(mod, inspect.isfunction): - register_check(function) -init_checks_registry() - - -class Checker(object): - """Load a Python source file, tokenize it, check coding style.""" - - def __init__(self, filename=None, lines=None, - options=None, report=None, **kwargs): - if options is None: - options = StyleGuide(kwargs).options - else: - assert not kwargs - self._io_error = None - self._physical_checks = options.physical_checks - self._logical_checks = options.logical_checks - self._ast_checks = options.ast_checks - self.max_line_length = options.max_line_length - self.multiline = False # in a multiline string? - self.hang_closing = options.hang_closing - self.verbose = options.verbose - self.filename = filename - # Dictionary where a checker can store its custom state. - self._checker_states = {} - if filename is None: - self.filename = 'stdin' - self.lines = lines or [] - elif filename == '-': - self.filename = 'stdin' - self.lines = stdin_get_value().splitlines(True) - elif lines is None: - try: - self.lines = readlines(filename) - except IOError: - (exc_type, exc) = sys.exc_info()[:2] - self._io_error = '%s: %s' % (exc_type.__name__, exc) - self.lines = [] - else: - self.lines = lines - if self.lines: - ord0 = ord(self.lines[0][0]) - if ord0 in (0xef, 0xfeff): # Strip the UTF-8 BOM - if ord0 == 0xfeff: - self.lines[0] = self.lines[0][1:] - elif self.lines[0][:3] == '\xef\xbb\xbf': - self.lines[0] = self.lines[0][3:] - self.report = report or options.report - self.report_error = self.report.error - - def report_invalid_syntax(self): - """Check if the syntax is valid.""" - (exc_type, exc) = sys.exc_info()[:2] - if len(exc.args) > 1: - offset = exc.args[1] - if len(offset) > 2: - offset = offset[1:3] - else: - offset = (1, 0) - self.report_error(offset[0], offset[1] or 0, - 'E901 %s: %s' % (exc_type.__name__, exc.args[0]), - self.report_invalid_syntax) - - def readline(self): - """Get the next line from the input buffer.""" - if self.line_number >= self.total_lines: - return '' - line = self.lines[self.line_number] - self.line_number += 1 - if self.indent_char is None and line[:1] in WHITESPACE: - self.indent_char = line[0] - return line - - def run_check(self, check, argument_names): - """Run a check plugin.""" - arguments = [] - for name in argument_names: - arguments.append(getattr(self, name)) - return check(*arguments) - - def init_checker_state(self, name, argument_names): - """ Prepares a custom state for the specific checker plugin.""" - if 'checker_state' in argument_names: - self.checker_state = self._checker_states.setdefault(name, {}) - - def check_physical(self, line): - """Run all physical checks on a raw input line.""" - self.physical_line = line - for name, check, argument_names in self._physical_checks: - self.init_checker_state(name, argument_names) - result = self.run_check(check, argument_names) - if result is not None: - (offset, text) = result - self.report_error(self.line_number, offset, text, check) - if text[:4] == 'E101': - self.indent_char = line[0] - - def build_tokens_line(self): - """Build a logical line from tokens.""" - logical = [] - comments = [] - length = 0 - prev_row = prev_col = mapping = None - for token_type, text, start, end, line in self.tokens: - if token_type in SKIP_TOKENS: - continue - if not mapping: - mapping = [(0, start)] - if token_type == tokenize.COMMENT: - comments.append(text) - continue - if token_type == tokenize.STRING: - text = mute_string(text) - if prev_row: - (start_row, start_col) = start - if prev_row != start_row: # different row - prev_text = self.lines[prev_row - 1][prev_col - 1] - if prev_text == ',' or (prev_text not in '{[(' and - text not in '}])'): - text = ' ' + text - elif prev_col != start_col: # different column - text = line[prev_col:start_col] + text - logical.append(text) - length += len(text) - mapping.append((length, end)) - (prev_row, prev_col) = end - self.logical_line = ''.join(logical) - self.noqa = comments and noqa(''.join(comments)) - return mapping - - def check_logical(self): - """Build a line from tokens and run all logical checks on it.""" - self.report.increment_logical_line() - mapping = self.build_tokens_line() - - if not mapping: - return - - (start_row, start_col) = mapping[0][1] - start_line = self.lines[start_row - 1] - self.indent_level = expand_indent(start_line[:start_col]) - if self.blank_before < self.blank_lines: - self.blank_before = self.blank_lines - if self.verbose >= 2: - print(self.logical_line[:80].rstrip()) - for name, check, argument_names in self._logical_checks: - if self.verbose >= 4: - print(' ' + name) - self.init_checker_state(name, argument_names) - for offset, text in self.run_check(check, argument_names) or (): - if not isinstance(offset, tuple): - for token_offset, pos in mapping: - if offset <= token_offset: - break - offset = (pos[0], pos[1] + offset - token_offset) - self.report_error(offset[0], offset[1], text, check) - if self.logical_line: - self.previous_indent_level = self.indent_level - self.previous_logical = self.logical_line - self.blank_lines = 0 - self.tokens = [] - - def check_ast(self): - """Build the file's AST and run all AST checks.""" - try: - tree = compile(''.join(self.lines), '', 'exec', PyCF_ONLY_AST) - except (ValueError, SyntaxError, TypeError): - return self.report_invalid_syntax() - for name, cls, __ in self._ast_checks: - checker = cls(tree, self.filename) - for lineno, offset, text, check in checker.run(): - if not self.lines or not noqa(self.lines[lineno - 1]): - self.report_error(lineno, offset, text, check) - - def generate_tokens(self): - """Tokenize the file, run physical line checks and yield tokens.""" - if self._io_error: - self.report_error(1, 0, 'E902 %s' % self._io_error, readlines) - tokengen = tokenize.generate_tokens(self.readline) - try: - for token in tokengen: - if token[2][0] > self.total_lines: - return - self.maybe_check_physical(token) - yield token - except (SyntaxError, tokenize.TokenError): - self.report_invalid_syntax() - - def maybe_check_physical(self, token): - """If appropriate (based on token), check current physical line(s).""" - # Called after every token, but act only on end of line. - if _is_eol_token(token): - # Obviously, a newline token ends a single physical line. - self.check_physical(token[4]) - elif token[0] == tokenize.STRING and '\n' in token[1]: - # Less obviously, a string that contains newlines is a - # multiline string, either triple-quoted or with internal - # newlines backslash-escaped. Check every physical line in the - # string *except* for the last one: its newline is outside of - # the multiline string, so we consider it a regular physical - # line, and will check it like any other physical line. - # - # Subtleties: - # - we don't *completely* ignore the last line; if it contains - # the magical "# noqa" comment, we disable all physical - # checks for the entire multiline string - # - have to wind self.line_number back because initially it - # points to the last line of the string, and we want - # check_physical() to give accurate feedback - if noqa(token[4]): - return - self.multiline = True - self.line_number = token[2][0] - for line in token[1].split('\n')[:-1]: - self.check_physical(line + '\n') - self.line_number += 1 - self.multiline = False - - def check_all(self, expected=None, line_offset=0): - """Run all checks on the input file.""" - self.report.init_file(self.filename, self.lines, expected, line_offset) - self.total_lines = len(self.lines) - if self._ast_checks: - self.check_ast() - self.line_number = 0 - self.indent_char = None - self.indent_level = self.previous_indent_level = 0 - self.previous_logical = '' - self.tokens = [] - self.blank_lines = self.blank_before = 0 - parens = 0 - for token in self.generate_tokens(): - self.tokens.append(token) - token_type, text = token[0:2] - if self.verbose >= 3: - if token[2][0] == token[3][0]: - pos = '[%s:%s]' % (token[2][1] or '', token[3][1]) - else: - pos = 'l.%s' % token[3][0] - print('l.%s\t%s\t%s\t%r' % - (token[2][0], pos, tokenize.tok_name[token[0]], text)) - if token_type == tokenize.OP: - if text in '([{': - parens += 1 - elif text in '}])': - parens -= 1 - elif not parens: - if token_type in NEWLINE: - if token_type == tokenize.NEWLINE: - self.check_logical() - self.blank_before = 0 - elif len(self.tokens) == 1: - # The physical line contains only this token. - self.blank_lines += 1 - del self.tokens[0] - else: - self.check_logical() - elif COMMENT_WITH_NL and token_type == tokenize.COMMENT: - if len(self.tokens) == 1: - # The comment also ends a physical line - token = list(token) - token[1] = text.rstrip('\r\n') - token[3] = (token[2][0], token[2][1] + len(token[1])) - self.tokens = [tuple(token)] - self.check_logical() - if self.tokens: - self.check_physical(self.lines[-1]) - self.check_logical() - return self.report.get_file_results() - - -class BaseReport(object): - """Collect the results of the checks.""" - - print_filename = False - - def __init__(self, options): - self._benchmark_keys = options.benchmark_keys - self._ignore_code = options.ignore_code - # Results - self.elapsed = 0 - self.total_errors = 0 - self.counters = dict.fromkeys(self._benchmark_keys, 0) - self.messages = {} - - def start(self): - """Start the timer.""" - self._start_time = time.time() - - def stop(self): - """Stop the timer.""" - self.elapsed = time.time() - self._start_time - - def init_file(self, filename, lines, expected, line_offset): - """Signal a new file.""" - self.filename = filename - self.lines = lines - self.expected = expected or () - self.line_offset = line_offset - self.file_errors = 0 - self.counters['files'] += 1 - self.counters['physical lines'] += len(lines) - - def increment_logical_line(self): - """Signal a new logical line.""" - self.counters['logical lines'] += 1 - - def error(self, line_number, offset, text, check): - """Report an error, according to options.""" - code = text[:4] - if self._ignore_code(code): - return - if code in self.counters: - self.counters[code] += 1 - else: - self.counters[code] = 1 - self.messages[code] = text[5:] - # Don't care about expected errors or warnings - if code in self.expected: - return - if self.print_filename and not self.file_errors: - print(self.filename) - self.file_errors += 1 - self.total_errors += 1 - return code - - def get_file_results(self): - """Return the count of errors and warnings for this file.""" - return self.file_errors - - def get_count(self, prefix=''): - """Return the total count of errors and warnings.""" - return sum([self.counters[key] - for key in self.messages if key.startswith(prefix)]) - - def get_statistics(self, prefix=''): - """Get statistics for message codes that start with the prefix. - - prefix='' matches all errors and warnings - prefix='E' matches all errors - prefix='W' matches all warnings - prefix='E4' matches all errors that have to do with imports - """ - return ['%-7s %s %s' % (self.counters[key], key, self.messages[key]) - for key in sorted(self.messages) if key.startswith(prefix)] - - def print_statistics(self, prefix=''): - """Print overall statistics (number of errors and warnings).""" - for line in self.get_statistics(prefix): - print(line) - - def print_benchmark(self): - """Print benchmark numbers.""" - print('%-7.2f %s' % (self.elapsed, 'seconds elapsed')) - if self.elapsed: - for key in self._benchmark_keys: - print('%-7d %s per second (%d total)' % - (self.counters[key] / self.elapsed, key, - self.counters[key])) - - -class FileReport(BaseReport): - """Collect the results of the checks and print only the filenames.""" - print_filename = True - - -class StandardReport(BaseReport): - """Collect and print the results of the checks.""" - - def __init__(self, options): - super(StandardReport, self).__init__(options) - self._fmt = REPORT_FORMAT.get(options.format.lower(), - options.format) - self._repeat = options.repeat - self._show_source = options.show_source - self._show_pep8 = options.show_pep8 - - def init_file(self, filename, lines, expected, line_offset): - """Signal a new file.""" - self._deferred_print = [] - return super(StandardReport, self).init_file( - filename, lines, expected, line_offset) - - def error(self, line_number, offset, text, check): - """Report an error, according to options.""" - code = super(StandardReport, self).error(line_number, offset, - text, check) - if code and (self.counters[code] == 1 or self._repeat): - self._deferred_print.append( - (line_number, offset, code, text[5:], check.__doc__)) - return code - - def get_file_results(self): - """Print the result and return the overall count for this file.""" - self._deferred_print.sort() - for line_number, offset, code, text, doc in self._deferred_print: - print(self._fmt % { - 'path': self.filename, - 'row': self.line_offset + line_number, 'col': offset + 1, - 'code': code, 'text': text, - }) - if self._show_source: - if line_number > len(self.lines): - line = '' - else: - line = self.lines[line_number - 1] - print(line.rstrip()) - print(re.sub(r'\S', ' ', line[:offset]) + '^') - if self._show_pep8 and doc: - print(' ' + doc.strip()) - - # stdout is block buffered when not stdout.isatty(). - # line can be broken where buffer boundary since other processes - # write to same file. - # flush() after print() to avoid buffer boundary. - # Typical buffer size is 8192. line written safely when - # len(line) < 8192. - sys.stdout.flush() - return self.file_errors - - -class DiffReport(StandardReport): - """Collect and print the results for the changed lines only.""" - - def __init__(self, options): - super(DiffReport, self).__init__(options) - self._selected = options.selected_lines - - def error(self, line_number, offset, text, check): - if line_number not in self._selected[self.filename]: - return - return super(DiffReport, self).error(line_number, offset, text, check) - - -class StyleGuide(object): - """Initialize a PEP-8 instance with few options.""" - - def __init__(self, *args, **kwargs): - # build options from the command line - self.checker_class = kwargs.pop('checker_class', Checker) - parse_argv = kwargs.pop('parse_argv', False) - config_file = kwargs.pop('config_file', False) - parser = kwargs.pop('parser', None) - # build options from dict - options_dict = dict(*args, **kwargs) - arglist = None if parse_argv else options_dict.get('paths', None) - options, self.paths = process_options( - arglist, parse_argv, config_file, parser) - if options_dict: - options.__dict__.update(options_dict) - if 'paths' in options_dict: - self.paths = options_dict['paths'] - - self.runner = self.input_file - self.options = options - - if not options.reporter: - options.reporter = BaseReport if options.quiet else StandardReport - - options.select = tuple(options.select or ()) - if not (options.select or options.ignore or - options.testsuite or options.doctest) and DEFAULT_IGNORE: - # The default choice: ignore controversial checks - options.ignore = tuple(DEFAULT_IGNORE.split(',')) - else: - # Ignore all checks which are not explicitly selected - options.ignore = ('',) if options.select else tuple(options.ignore) - options.benchmark_keys = BENCHMARK_KEYS[:] - options.ignore_code = self.ignore_code - options.physical_checks = self.get_checks('physical_line') - options.logical_checks = self.get_checks('logical_line') - options.ast_checks = self.get_checks('tree') - self.init_report() - - def init_report(self, reporter=None): - """Initialize the report instance.""" - self.options.report = (reporter or self.options.reporter)(self.options) - return self.options.report - - def check_files(self, paths=None): - """Run all checks on the paths.""" - if paths is None: - paths = self.paths - report = self.options.report - runner = self.runner - report.start() - try: - for path in paths: - if os.path.isdir(path): - self.input_dir(path) - elif not self.excluded(path): - runner(path) - except KeyboardInterrupt: - print('... stopped') - report.stop() - return report - - def input_file(self, filename, lines=None, expected=None, line_offset=0): - """Run all checks on a Python source file.""" - if self.options.verbose: - print('checking %s' % filename) - fchecker = self.checker_class( - filename, lines=lines, options=self.options) - return fchecker.check_all(expected=expected, line_offset=line_offset) - - def input_dir(self, dirname): - """Check all files in this directory and all subdirectories.""" - dirname = dirname.rstrip('/') - if self.excluded(dirname): - return 0 - counters = self.options.report.counters - verbose = self.options.verbose - filepatterns = self.options.filename - runner = self.runner - for root, dirs, files in os.walk(dirname): - if verbose: - print('directory ' + root) - counters['directories'] += 1 - for subdir in sorted(dirs): - if self.excluded(subdir, root): - dirs.remove(subdir) - for filename in sorted(files): - # contain a pattern that matches? - if ((filename_match(filename, filepatterns) and - not self.excluded(filename, root))): - runner(os.path.join(root, filename)) - - def excluded(self, filename, parent=None): - """Check if the file should be excluded. - - Check if 'options.exclude' contains a pattern that matches filename. - """ - if not self.options.exclude: - return False - basename = os.path.basename(filename) - if filename_match(basename, self.options.exclude): - return True - if parent: - filename = os.path.join(parent, filename) - filename = os.path.abspath(filename) - return filename_match(filename, self.options.exclude) - - def ignore_code(self, code): - """Check if the error code should be ignored. - - If 'options.select' contains a prefix of the error code, - return False. Else, if 'options.ignore' contains a prefix of - the error code, return True. - """ - if len(code) < 4 and any(s.startswith(code) - for s in self.options.select): - return False - return (code.startswith(self.options.ignore) and - not code.startswith(self.options.select)) - - def get_checks(self, argument_name): - """Get all the checks for this category. - - Find all globally visible functions where the first argument name - starts with argument_name and which contain selected tests. - """ - checks = [] - for check, attrs in _checks[argument_name].items(): - (codes, args) = attrs - if any(not (code and self.ignore_code(code)) for code in codes): - checks.append((check.__name__, check, args)) - return sorted(checks) - - -def get_parser(prog='pep8', version=__version__): - parser = OptionParser(prog=prog, version=version, - usage="%prog [options] input ...") - parser.config_options = [ - 'exclude', 'filename', 'select', 'ignore', 'max-line-length', - 'hang-closing', 'count', 'format', 'quiet', 'show-pep8', - 'show-source', 'statistics', 'verbose'] - parser.add_option('-v', '--verbose', default=0, action='count', - help="print status messages, or debug with -vv") - parser.add_option('-q', '--quiet', default=0, action='count', - help="report only file names, or nothing with -qq") - parser.add_option('-r', '--repeat', default=True, action='store_true', - help="(obsolete) show all occurrences of the same error") - parser.add_option('--first', action='store_false', dest='repeat', - help="show first occurrence of each error") - parser.add_option('--exclude', metavar='patterns', default=DEFAULT_EXCLUDE, - help="exclude files or directories which match these " - "comma separated patterns (default: %default)") - parser.add_option('--filename', metavar='patterns', default='*.py', - help="when parsing directories, only check filenames " - "matching these comma separated patterns " - "(default: %default)") - parser.add_option('--select', metavar='errors', default='', - help="select errors and warnings (e.g. E,W6)") - parser.add_option('--ignore', metavar='errors', default='', - help="skip errors and warnings (e.g. E4,W) " - "(default: %s)" % DEFAULT_IGNORE) - parser.add_option('--show-source', action='store_true', - help="show source code for each error") - parser.add_option('--show-pep8', action='store_true', - help="show text of PEP 8 for each error " - "(implies --first)") - parser.add_option('--statistics', action='store_true', - help="count errors and warnings") - parser.add_option('--count', action='store_true', - help="print total number of errors and warnings " - "to standard error and set exit code to 1 if " - "total is not null") - parser.add_option('--max-line-length', type='int', metavar='n', - default=MAX_LINE_LENGTH, - help="set maximum allowed line length " - "(default: %default)") - parser.add_option('--hang-closing', action='store_true', - help="hang closing bracket instead of matching " - "indentation of opening bracket's line") - parser.add_option('--format', metavar='format', default='default', - help="set the error format [default|pylint|]") - parser.add_option('--diff', action='store_true', - help="report changes only within line number ranges in " - "the unified diff received on STDIN") - group = parser.add_option_group("Testing Options") - if os.path.exists(TESTSUITE_PATH): - group.add_option('--testsuite', metavar='dir', - help="run regression tests from dir") - group.add_option('--doctest', action='store_true', - help="run doctest on myself") - group.add_option('--benchmark', action='store_true', - help="measure processing speed") - return parser - - -def read_config(options, args, arglist, parser): - """Read and parse configurations - - If a config file is specified on the command line with the "--config" - option, then only it is used for configuration. - - Otherwise, the user configuration (~/.config/pep8) and any local - configurations in the current directory or above will be merged together - (in that order) using the read method of ConfigParser. - """ - config = RawConfigParser() - - cli_conf = options.config - - local_dir = os.curdir - - if USER_CONFIG and os.path.isfile(USER_CONFIG): - if options.verbose: - print('user configuration: %s' % USER_CONFIG) - config.read(USER_CONFIG) - - parent = tail = args and os.path.abspath(os.path.commonprefix(args)) - while tail: - if config.read(os.path.join(parent, fn) for fn in PROJECT_CONFIG): - local_dir = parent - if options.verbose: - print('local configuration: in %s' % parent) - break - (parent, tail) = os.path.split(parent) - - if cli_conf and os.path.isfile(cli_conf): - if options.verbose: - print('cli configuration: %s' % cli_conf) - config.read(cli_conf) - - pep8_section = parser.prog - if config.has_section(pep8_section): - option_list = dict([(o.dest, o.type or o.action) - for o in parser.option_list]) - - # First, read the default values - (new_options, __) = parser.parse_args([]) - - # Second, parse the configuration - for opt in config.options(pep8_section): - if opt.replace('_', '-') not in parser.config_options: - print(" unknown option '%s' ignored" % opt) - continue - if options.verbose > 1: - print(" %s = %s" % (opt, config.get(pep8_section, opt))) - normalized_opt = opt.replace('-', '_') - opt_type = option_list[normalized_opt] - if opt_type in ('int', 'count'): - value = config.getint(pep8_section, opt) - elif opt_type == 'string': - value = config.get(pep8_section, opt) - if normalized_opt == 'exclude': - value = normalize_paths(value, local_dir) - else: - assert opt_type in ('store_true', 'store_false') - value = config.getboolean(pep8_section, opt) - setattr(new_options, normalized_opt, value) - - # Third, overwrite with the command-line options - (options, __) = parser.parse_args(arglist, values=new_options) - options.doctest = options.testsuite = False - return options - - -def process_options(arglist=None, parse_argv=False, config_file=None, - parser=None): - """Process options passed either via arglist or via command line args. - - Passing in the ``config_file`` parameter allows other tools, such as flake8 - to specify their own options to be processed in pep8. - """ - if not parser: - parser = get_parser() - if not parser.has_option('--config'): - group = parser.add_option_group("Configuration", description=( - "The project options are read from the [%s] section of the " - "tox.ini file or the setup.cfg file located in any parent folder " - "of the path(s) being processed. Allowed options are: %s." % - (parser.prog, ', '.join(parser.config_options)))) - group.add_option('--config', metavar='path', default=config_file, - help="user config file location") - # Don't read the command line if the module is used as a library. - if not arglist and not parse_argv: - arglist = [] - # If parse_argv is True and arglist is None, arguments are - # parsed from the command line (sys.argv) - (options, args) = parser.parse_args(arglist) - options.reporter = None - - if options.ensure_value('testsuite', False): - args.append(options.testsuite) - elif not options.ensure_value('doctest', False): - if parse_argv and not args: - if options.diff or any(os.path.exists(name) - for name in PROJECT_CONFIG): - args = ['.'] - else: - parser.error('input not specified') - options = read_config(options, args, arglist, parser) - options.reporter = parse_argv and options.quiet == 1 and FileReport - - options.filename = _parse_multi_options(options.filename) - options.exclude = normalize_paths(options.exclude) - options.select = _parse_multi_options(options.select) - options.ignore = _parse_multi_options(options.ignore) - - if options.diff: - options.reporter = DiffReport - stdin = stdin_get_value() - options.selected_lines = parse_udiff(stdin, options.filename, args[0]) - args = sorted(options.selected_lines) - - return options, args - - -def _parse_multi_options(options, split_token=','): - r"""Split and strip and discard empties. - - Turns the following: - - A, - B, - - into ["A", "B"] - """ - if options: - return [o.strip() for o in options.split(split_token) if o.strip()] - else: - return options - - -def _main(): - """Parse options and run checks on Python source.""" - import signal - - # Handle "Broken pipe" gracefully - try: - signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1)) - except AttributeError: - pass # not supported on Windows - - pep8style = StyleGuide(parse_argv=True) - options = pep8style.options - - if options.doctest or options.testsuite: - from testsuite.support import run_tests - report = run_tests(pep8style) - else: - report = pep8style.check_files() - - if options.statistics: - report.print_statistics() - - if options.benchmark: - report.print_benchmark() - - if options.testsuite and not options.quiet: - report.print_results() - - if report.total_errors: - if options.count: - sys.stderr.write(str(report.total_errors) + '\n') - sys.exit(1) - -if __name__ == '__main__': - _main() diff --git a/Lib/site.py b/Lib/site.py index 29b19bb..56ba709 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -623,59 +623,3 @@ def _script(): if __name__ == '__main__': _script() - - -def no_chocolate(): - import io - import pep8 - import random - import shutil - import tokenize - - _builtin_compile = builtins.compile - words = ('chocolate', 'glory', 'fun', 'spam', 'love', 'guts') - pep8style = pep8.StyleGuide() - - def compile_pep8(source, filename, mode, **kw): - name = os.path.splitext(os.path.basename(filename))[0] - if not name.endswith('_noqa'): - bio = io.BytesIO(source) - encoding = tokenize.detect_encoding(bio.readline)[0] - lines = source.decode(encoding).splitlines(True) - - report = pep8.StandardReport(options=pep8style.options) - checker = pep8.Checker(filename, lines, - report=report, - options=pep8style.options) - checker.check_all() - if report.total_errors: - word = random.choice(words) - raise ImportError("no pep8, no %s" % word) - return _builtin_compile(source, filename, mode, **kw) - - builtins.compile = compile_pep8 - - # remove precompiled .pyc created during the bootstrap, - # to run PEP 8 checks on .py files - libdir_cache = os.path.join(os.path.dirname(__file__), '__pycache__') - try: - shutil.rmtree(libdir_cache) - except: - pass - - for name in sorted(sys.modules): - # Minimum to be able to import modules - if name in {'builtins', 'importlib._bootstrap', - 'importlib._bootstrap_external', 'importlib', - 'importlib.machinery', '__main__', 'io', 'sys', 'site'}: - continue - del sys.modules[name] - - -try: - import _ssl -except ImportError: - # Python not bootstraped yet - pass -else: - no_chocolate() -- cgit v0.12 From dcfebb32e277a68b9c6582e6a0484e6dc24e9b66 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Fri, 1 Apr 2016 06:55:55 +0000 Subject: Issue #26676: Add missing XMLPullParser to ElementTree.__all__ --- Doc/whatsnew/3.6.rst | 3 ++- Lib/test/test_xml_etree.py | 6 ++++-- Lib/xml/etree/ElementTree.py | 2 +- Misc/NEWS | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 928d748..6799f69 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -460,7 +460,8 @@ Changes in the Python API * The following modules have had missing APIs added to their :attr:`__all__` attributes to match the documented APIs: :mod:`calendar`, :mod:`csv`, - :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, + :mod:`~xml.etree.ElementTree`, :mod:`enum`, + :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`optparse`, :mod:`tarfile`, :mod:`threading` and :mod:`wave`. This means they will export new symbols when ``import *`` is used. See :issue:`23883`. diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index bfd7215..a56f158 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -91,8 +91,6 @@ ENTITY_XML = """\ class ModuleTest(unittest.TestCase): - # TODO: this should be removed once we get rid of the global module vars - def test_sanity(self): # Import sanity. @@ -100,6 +98,10 @@ class ModuleTest(unittest.TestCase): from xml.etree import ElementInclude from xml.etree import ElementPath + def test_all(self): + names = ("xml.etree.ElementTree", "_elementtree") + support.check__all__(self, ET, names, blacklist=("HTML_EMPTY",)) + def serialize(elem, to_string=True, encoding='unicode', **options): if encoding != 'unicode': diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index b92fb52..50e42c8 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -85,7 +85,7 @@ __all__ = [ "TreeBuilder", "VERSION", "XML", "XMLID", - "XMLParser", + "XMLParser", "XMLPullParser", "register_namespace", ] diff --git a/Misc/NEWS b/Misc/NEWS index 6497701..8a7fac4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,8 @@ Core and Builtins Library ------- +- Issue #26676: Added missing XMLPullParser to ElementTree.__all__. + - Issue #22854: Change BufferedReader.writable() and BufferedWriter.readable() to always return False. -- cgit v0.12 From 50badad807abc5367359bd81a2a8051ff5cdce7e Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 3 Apr 2016 01:28:53 +0000 Subject: Issue #26586: Simple enhancements to BaseHTTPRequestHandler by Xiang Zhang --- Lib/http/server.py | 28 +++++++++++++--------------- Lib/test/test_httpservers.py | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index de6b531..f4ad260 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -137,7 +137,7 @@ class HTTPServer(socketserver.TCPServer): def server_bind(self): """Override server_bind to store the server name.""" socketserver.TCPServer.server_bind(self) - host, port = self.socket.getsockname()[:2] + host, port = self.server_address[:2] self.server_name = socket.getfqdn(host) self.server_port = port @@ -283,12 +283,9 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): words = requestline.split() if len(words) == 3: command, path, version = words - if version[:5] != 'HTTP/': - self.send_error( - HTTPStatus.BAD_REQUEST, - "Bad request version (%r)" % version) - return False try: + if version[:5] != 'HTTP/': + raise ValueError base_version_number = version.split('/', 1)[1] version_number = base_version_number.split(".") # RFC 2145 section 3.1 says there can be only one "." and @@ -310,7 +307,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): if version_number >= (2, 0): self.send_error( HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, - "Invalid HTTP Version (%s)" % base_version_number) + "Invalid HTTP version (%s)" % base_version_number) return False elif len(words) == 2: command, path = words @@ -333,10 +330,11 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): try: self.headers = http.client.parse_headers(self.rfile, _class=self.MessageClass) - except http.client.LineTooLong: + except http.client.LineTooLong as err: self.send_error( - HTTPStatus.BAD_REQUEST, - "Line too long") + HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE, + "Line too long", + str(err)) return False except http.client.HTTPException as err: self.send_error( @@ -482,12 +480,12 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): def send_response_only(self, code, message=None): """Send the response header only.""" - if message is None: - if code in self.responses: - message = self.responses[code][0] - else: - message = '' if self.request_version != 'HTTP/0.9': + if message is None: + if code in self.responses: + message = self.responses[code][0] + else: + message = '' if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append(("%s %d %s\r\n" % diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 98798ae..d9e4bf0 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -855,7 +855,7 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): # Issue #6791: same for headers result = self.send_typical_request( b'GET / HTTP/1.1\r\nX-Foo: bar' + b'r' * 65537 + b'\r\n\r\n') - self.assertEqual(result[0], b'HTTP/1.1 400 Line too long\r\n') + self.assertEqual(result[0], b'HTTP/1.1 431 Line too long\r\n') self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') -- cgit v0.12 From 519f91215bd353c745946e5d892fd1f089dbcb84 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 3 Apr 2016 02:12:54 +0000 Subject: Issue #25951: Fix SSLSocket.sendall() to return None, by Aviv Palivoda --- Lib/ssl.py | 1 - Lib/test/test_ssl.py | 16 ++++++++++------ Misc/NEWS | 3 +++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 65ad38f..68db748 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -886,7 +886,6 @@ class SSLSocket(socket): while (count < amount): v = self.send(data[count:]) count += v - return amount else: return socket.sendall(self, data, flags) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index e0f231c..00d437a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -2709,12 +2709,13 @@ if _have_threads: count, addr = s.recvfrom_into(b) return b[:count] - # (name, method, whether to expect success, *args) + # (name, method, expect success?, *args, return value func) send_methods = [ - ('send', s.send, True, []), - ('sendto', s.sendto, False, ["some.address"]), - ('sendall', s.sendall, True, []), + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), ] + # (name, method, whether to expect success, *args) recv_methods = [ ('recv', s.recv, True, []), ('recvfrom', s.recvfrom, False, ["some.address"]), @@ -2723,10 +2724,13 @@ if _have_threads: ] data_prefix = "PREFIX_" - for meth_name, send_meth, expect_success, args in send_methods: + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: indata = (data_prefix + meth_name).encode('ascii') try: - send_meth(indata, *args) + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) outdata = s.read() if outdata != indata.lower(): self.fail( diff --git a/Misc/NEWS b/Misc/NEWS index cba8466..f66014a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #25951: Change SSLSocket.sendall() to return None, as explicitly + documented for plain socket objects. Patch by Aviv Palivoda. + - Issue #26586: In http.server, respond with "413 Request header fields too large" if there are too many header fields to parse, rather than killing the connection and raising an unhandled exception. Patch by Xiang Zhang. -- cgit v0.12 From 16ca06b8cb2426b540fdab75914d7cd0f715b7f0 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 4 Apr 2016 10:59:29 -0700 Subject: Add collections.Reversible. Patch by Ivan Levkivskyi. Fixes issue #25987. --- Doc/library/collections.abc.rst | 7 ++++++- Doc/library/typing.rst | 11 +++++------ Lib/_collections_abc.py | 23 +++++++++++++++++++++-- Lib/test/test_collections.py | 31 ++++++++++++++++++++++++++++--- Lib/test/test_functools.py | 2 +- 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index d9b93ad..608641b 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -40,12 +40,13 @@ ABC Inherits from Abstract Methods Mixin :class:`Hashable` ``__hash__`` :class:`Iterable` ``__iter__`` :class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` +:class:`Reversible` :class:`Iterable` ``__reversed__`` :class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` :class:`Sized` ``__len__`` :class:`Callable` ``__call__`` :class:`Sequence` :class:`Sized`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Iterable`, ``__len__`` ``index``, and ``count`` + :class:`Reversible`, ``__len__`` ``index``, and ``count`` :class:`Container` :class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and @@ -107,6 +108,10 @@ ABC Inherits from Abstract Methods Mixin :meth:`~iterator.__next__` methods. See also the definition of :term:`iterator`. +.. class:: Reversible + + ABC for classes that provide the :meth:`__reversed__` method. + .. class:: Generator ABC for generator classes that implement the protocol defined in diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 12b5490..1bd4b09 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -351,6 +351,10 @@ The module defines the following classes, functions and decorators: A generic version of the :class:`collections.abc.Iterator`. +.. class:: Reversible(Iterable[T_co]) + + A generic version of the :class:`collections.abc.Reversible`. + .. class:: SupportsInt An ABC with one abstract method ``__int__``. @@ -369,11 +373,6 @@ The module defines the following classes, functions and decorators: An ABC with one abstract method ``__round__`` that is covariant in its return type. -.. class:: Reversible - - An ABC with one abstract method ``__reversed__`` returning - an ``Iterator[T_co]``. - .. class:: Container(Generic[T_co]) A generic version of :class:`collections.abc.Container`. @@ -394,7 +393,7 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.MutableMapping`. -.. class:: Sequence(Sized, Iterable[T_co], Container[T_co]) +.. class:: Sequence(Sized, Reversible[T_co], Container[T_co]) A generic version of :class:`collections.abc.Sequence`. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index f89bb6f..d337584 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -10,7 +10,7 @@ from abc import ABCMeta, abstractmethod import sys __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", - "Hashable", "Iterable", "Iterator", "Generator", + "Hashable", "Iterable", "Iterator", "Generator", "Reversible", "Sized", "Container", "Callable", "Set", "MutableSet", "Mapping", "MutableMapping", @@ -240,6 +240,25 @@ Iterator.register(tuple_iterator) Iterator.register(zip_iterator) +class Reversible(Iterable): + + __slots__ = () + + @abstractmethod + def __reversed__(self): + return NotImplemented + + @classmethod + def __subclasshook__(cls, C): + if cls is Reversible: + for B in C.__mro__: + if "__reversed__" in B.__dict__: + if B.__dict__["__reversed__"] is not None: + return True + break + return NotImplemented + + class Generator(Iterator): __slots__ = () @@ -794,7 +813,7 @@ MutableMapping.register(dict) ### SEQUENCES ### -class Sequence(Sized, Iterable, Container): +class Sequence(Sized, Reversible, Container): """All the operations on a read-only sequence. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 4c32e09..4202462 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -20,7 +20,7 @@ from collections import UserDict, UserString, UserList from collections import ChainMap from collections import deque from collections.abc import Awaitable, Coroutine, AsyncIterator, AsyncIterable -from collections.abc import Hashable, Iterable, Iterator, Generator +from collections.abc import Hashable, Iterable, Iterator, Generator, Reversible from collections.abc import Sized, Container, Callable from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView @@ -689,6 +689,31 @@ class TestOneTrickPonyABCs(ABCTestCase): self.validate_abstract_methods(Iterable, '__iter__') self.validate_isinstance(Iterable, '__iter__') + def test_Reversible(self): + # Check some non-reversibles + non_samples = [None, 42, 3.14, 1j, dict(), set(), frozenset()] + for x in non_samples: + self.assertNotIsInstance(x, Reversible) + self.assertFalse(issubclass(type(x), Reversible), repr(type(x))) + # Check some reversibles + samples = [tuple(), list()] + for x in samples: + self.assertIsInstance(x, Reversible) + self.assertTrue(issubclass(type(x), Reversible), repr(type(x))) + # Check also Mapping, MutableMapping, and Sequence + self.assertTrue(issubclass(Sequence, Reversible), repr(Sequence)) + self.assertFalse(issubclass(Mapping, Reversible), repr(Mapping)) + self.assertFalse(issubclass(MutableMapping, Reversible), repr(MutableMapping)) + # Check direct subclassing + class R(Reversible): + def __iter__(self): + return iter(list()) + def __reversed__(self): + return iter(list()) + self.assertEqual(list(reversed(R())), []) + self.assertFalse(issubclass(float, R)) + self.validate_abstract_methods(Reversible, '__reversed__', '__iter__') + def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: @@ -842,14 +867,14 @@ class TestOneTrickPonyABCs(ABCTestCase): self.validate_isinstance(Callable, '__call__') def test_direct_subclassing(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C(B): pass self.assertTrue(issubclass(C, B)) self.assertFalse(issubclass(int, C)) def test_registration(self): - for B in Hashable, Iterable, Iterator, Sized, Container, Callable: + for B in Hashable, Iterable, Iterator, Reversible, Sized, Container, Callable: class C: __hash__ = None # Make sure it isn't hashable by default self.assertFalse(issubclass(C, B), B.__name__) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 31930fc..a22d2e6 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1516,7 +1516,7 @@ class TestSingleDispatch(unittest.TestCase): m = mro(D, bases) self.assertEqual(m, [D, c.MutableSequence, c.Sequence, c.defaultdict, dict, c.MutableMapping, - c.Mapping, c.Sized, c.Iterable, c.Container, + c.Mapping, c.Sized, c.Reversible, c.Iterable, c.Container, object]) # Container and Callable are registered on different base classes and -- cgit v0.12 From 819399b2ab3cba99de540b07f6ec12b2777f6ec0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 22:17:52 +0300 Subject: Issue #26671: Enhanced path_converter. Exceptions raised during converting argument of correct type are no longer overridded with TypeError. Some error messages are now more detailed. --- Modules/posixmodule.c | 106 ++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 56 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9013888..e6704ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -669,21 +669,20 @@ _Py_Dev_Converter(PyObject *obj, void *p) #endif static int -_fd_converter(PyObject *o, int *p, const char *allowed) +_fd_converter(PyObject *o, int *p) { int overflow; long long_value; PyObject *index = PyNumber_Index(o); if (index == NULL) { - PyErr_Format(PyExc_TypeError, - "argument should be %s, not %.200s", - allowed, Py_TYPE(o)->tp_name); return 0; } + assert(PyLong_Check(index)); long_value = PyLong_AsLongAndOverflow(index, &overflow); Py_DECREF(index); + assert(!PyErr_Occurred()); if (overflow > 0 || long_value > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "fd is greater than maximum"); @@ -706,7 +705,15 @@ dir_fd_converter(PyObject *o, void *p) *(int *)p = DEFAULT_DIR_FD; return 1; } - return _fd_converter(o, (int *)p, "integer"); + else if (PyIndex_Check(o)) { + return _fd_converter(o, (int *)p); + } + else { + PyErr_Format(PyExc_TypeError, + "argument should be integer or None, not %.200s", + Py_TYPE(o)->tp_name); + return 0; + } } @@ -816,9 +823,10 @@ path_cleanup(path_t *path) { } static int -path_converter(PyObject *o, void *p) { +path_converter(PyObject *o, void *p) +{ path_t *path = (path_t *)p; - PyObject *unicode, *bytes; + PyObject *bytes; Py_ssize_t length; char *narrow; @@ -837,12 +845,7 @@ path_converter(PyObject *o, void *p) { /* ensure it's always safe to call path_cleanup() */ path->cleanup = NULL; - if (o == Py_None) { - if (!path->nullable) { - FORMAT_EXCEPTION(PyExc_TypeError, - "can't specify None for %s argument"); - return 0; - } + if ((o == Py_None) && path->nullable) { path->wide = NULL; path->narrow = NULL; path->length = 0; @@ -851,24 +854,20 @@ path_converter(PyObject *o, void *p) { return 1; } - unicode = PyUnicode_FromObject(o); - if (unicode) { + if (PyUnicode_Check(o)) { #ifdef MS_WINDOWS wchar_t *wide; - wide = PyUnicode_AsUnicodeAndSize(unicode, &length); + wide = PyUnicode_AsUnicodeAndSize(o, &length); if (!wide) { - Py_DECREF(unicode); return 0; } if (length > 32767) { FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - Py_DECREF(unicode); return 0; } if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character"); - Py_DECREF(unicode); + FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); return 0; } @@ -877,51 +876,46 @@ path_converter(PyObject *o, void *p) { path->length = length; path->object = o; path->fd = -1; - path->cleanup = unicode; - return Py_CLEANUP_SUPPORTED; + return 1; #else - int converted = PyUnicode_FSConverter(unicode, &bytes); - Py_DECREF(unicode); - if (!converted) - bytes = NULL; + if (!PyUnicode_FSConverter(o, &bytes)) { + return 0; + } #endif } - else { - PyErr_Clear(); - if (PyObject_CheckBuffer(o)) - bytes = PyBytes_FromObject(o); - else - bytes = NULL; + else if (PyObject_CheckBuffer(o)) { +# ifdef MS_WINDOWS + if (win32_warn_bytes_api()) { + return 0; + } +# endif + bytes = PyBytes_FromObject(o); if (!bytes) { - PyErr_Clear(); - if (path->allow_fd) { - int fd; - int result = _fd_converter(o, &fd, - "string, bytes or integer"); - if (result) { - path->wide = NULL; - path->narrow = NULL; - path->length = 0; - path->object = o; - path->fd = fd; - return result; - } - } + return 0; } } - - if (!bytes) { - if (!PyErr_Occurred()) - FORMAT_EXCEPTION(PyExc_TypeError, "illegal type for %s parameter"); - return 0; + else if (path->allow_fd && PyIndex_Check(o)) { + if (!_fd_converter(o, &path->fd)) { + return 0; + } + path->wide = NULL; + path->narrow = NULL; + path->length = 0; + path->object = o; + return 1; } - -#ifdef MS_WINDOWS - if (win32_warn_bytes_api()) { - Py_DECREF(bytes); + else { + PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", + path->function_name ? path->function_name : "", + path->function_name ? ": " : "", + path->argument_name ? path->argument_name : "path", + path->allow_fd && path->nullable ? "string, bytes, integer or None" : + path->allow_fd ? "string, bytes or integer" : + path->nullable ? "string, bytes or None" : + "string or bytes", + Py_TYPE(o)->tp_name); return 0; } -#endif length = PyBytes_GET_SIZE(bytes); #ifdef MS_WINDOWS -- cgit v0.12 From 026110f0a2ce027a781e429ee9a0ed14f9b2bc4a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 22:55:31 +0300 Subject: Issue #26671: Fixed #ifdef indentation. --- Modules/posixmodule.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e6704ac..bc41d93 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -# ifdef MS_WINDOWS +#ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -# endif +#endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; @@ -905,6 +905,30 @@ path_converter(PyObject *o, void *p) return 1; } else { + PyObject *pathattr; + _Py_IDENTIFIER(path); + + pathattr = _PyObject_GetAttrId(o, &PyId_path); + if (pathattr == NULL) { + PyErr_Clear(); + } + else if (PyUnicode_Check(pathattr) || PyObject_CheckBuffer(pathattr)) { + if (!path_converter(pathattr, path)) { + Py_DECREF(pathattr); + return 0; + } + if (path->cleanup == NULL) { + path->cleanup = pathattr; + } + else { + Py_DECREF(pathattr); + } + return Py_CLEANUP_SUPPORTED; + } + else { + Py_DECREF(pathattr); + } + PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", path->function_name ? ": " : "", -- cgit v0.12 From aaf553bac4124cda748b68caf7d501fd5b574046 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 23:02:25 +0300 Subject: Backed out changeset 8dc144e47252 --- Modules/posixmodule.c | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bc41d93..e6704ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -#ifdef MS_WINDOWS +# ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -#endif +# endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; @@ -905,30 +905,6 @@ path_converter(PyObject *o, void *p) return 1; } else { - PyObject *pathattr; - _Py_IDENTIFIER(path); - - pathattr = _PyObject_GetAttrId(o, &PyId_path); - if (pathattr == NULL) { - PyErr_Clear(); - } - else if (PyUnicode_Check(pathattr) || PyObject_CheckBuffer(pathattr)) { - if (!path_converter(pathattr, path)) { - Py_DECREF(pathattr); - return 0; - } - if (path->cleanup == NULL) { - path->cleanup = pathattr; - } - else { - Py_DECREF(pathattr); - } - return Py_CLEANUP_SUPPORTED; - } - else { - Py_DECREF(pathattr); - } - PyErr_Format(PyExc_TypeError, "%s%s%s should be %s, not %.200s", path->function_name ? path->function_name : "", path->function_name ? ": " : "", -- cgit v0.12 From 3291d85a2f6c6791ff1d8d1111b6bc46bf09d9e4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 Apr 2016 23:02:46 +0300 Subject: Issue #26671: Fixed #ifdef indentation. --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e6704ac..a87bbbd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -884,11 +884,11 @@ path_converter(PyObject *o, void *p) #endif } else if (PyObject_CheckBuffer(o)) { -# ifdef MS_WINDOWS +#ifdef MS_WINDOWS if (win32_warn_bytes_api()) { return 0; } -# endif +#endif bytes = PyBytes_FromObject(o); if (!bytes) { return 0; -- cgit v0.12 From 7155b881f249dc9c72cdcea53b2a945798783781 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 8 Apr 2016 08:48:20 +0300 Subject: Issue #26671: Fixed tests for changed error messages. --- Lib/test/test_posix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 2a59c38..28cdd90 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -411,7 +411,7 @@ class PosixTester(unittest.TestCase): self.assertTrue(posix.stat(bytearray(os.fsencode(support.TESTFN)))) self.assertRaisesRegex(TypeError, - 'can\'t specify None for path argument', + 'should be string, bytes or integer, not', posix.stat, None) self.assertRaisesRegex(TypeError, 'should be string, bytes or integer, not', @@ -863,9 +863,9 @@ class PosixTester(unittest.TestCase): self.assertEqual(s1, s2) s2 = posix.stat(support.TESTFN, dir_fd=None) self.assertEqual(s1, s2) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=posix.getcwd()) - self.assertRaisesRegex(TypeError, 'should be integer, not', + self.assertRaisesRegex(TypeError, 'should be integer or None, not', posix.stat, support.TESTFN, dir_fd=float(f)) self.assertRaises(OverflowError, posix.stat, support.TESTFN, dir_fd=10**20) -- cgit v0.12 From 9e080e0e741dd70cf86500f848eee19cf8b29efa Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:15:27 -0700 Subject: Issue #25609: Introduce contextlib.AbstractContextManager and typing.ContextManager. --- Doc/library/contextlib.rst | 16 ++++++++++++++-- Doc/library/typing.rst | 12 +++++++++--- Doc/whatsnew/3.6.rst | 26 +++++++++++++++++++++++--- Lib/contextlib.py | 42 +++++++++++++++++++++++++++++++----------- Lib/test/test_contextlib.py | 33 +++++++++++++++++++++++++++++++++ Lib/test/test_typing.py | 18 ++++++++++++++++++ Lib/typing.py | 7 +++++++ Misc/NEWS | 3 +++ 8 files changed, 138 insertions(+), 19 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index c112241..7876e7a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -18,6 +18,18 @@ Utilities Functions and classes provided: +.. class:: AbstractContextManager + + An abstract base class for classes that implement + :meth:`object.__enter__` and :meth:`object.__exit__`. A default + implementation for :meth:`object.__enter__` is provided which returns + ``self`` while :meth:`object.__exit__` is an abstract method which by default + returns ``None``. See also the definition of :ref:`typecontextmanager`. + + .. versionadded:: 3.6 + + + .. decorator:: contextmanager This function is a :term:`decorator` that can be used to define a factory @@ -447,9 +459,9 @@ Here's an example of doing this for a context manager that accepts resource acquisition and release functions, along with an optional validation function, and maps them to the context management protocol:: - from contextlib import contextmanager, ExitStack + from contextlib import contextmanager, AbstractContextManager, ExitStack - class ResourceManager: + class ResourceManager(AbstractContextManager): def __init__(self, acquire_resource, release_resource, check_resource_ok=None): self.acquire_resource = acquire_resource diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 1bd4b09..f609a51 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -345,15 +345,15 @@ The module defines the following classes, functions and decorators: .. class:: Iterable(Generic[T_co]) - A generic version of the :class:`collections.abc.Iterable`. + A generic version of :class:`collections.abc.Iterable`. .. class:: Iterator(Iterable[T_co]) - A generic version of the :class:`collections.abc.Iterator`. + A generic version of :class:`collections.abc.Iterator`. .. class:: Reversible(Iterable[T_co]) - A generic version of the :class:`collections.abc.Reversible`. + A generic version of :class:`collections.abc.Reversible`. .. class:: SupportsInt @@ -448,6 +448,12 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.abc.ValuesView`. +.. class:: ContextManager(Generic[T_co]) + + A generic version of :class:`contextlib.AbstractContextManager`. + + .. versionadded:: 3.6 + .. class:: Dict(dict, MutableMapping[KT, VT]) A generic version of :class:`dict`. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 6799f69..555814e 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -190,6 +190,18 @@ New Modules Improved Modules ================ +contextlib +---------- + +The :class:`contextlib.AbstractContextManager` class has been added to +provide an abstract base class for context managers. It provides a +sensible default implementation for `__enter__()` which returns +`self` and leaves `__exit__()` an abstract method. A matching +class has been added to the :mod:`typing` module as +:class:`typing.ContextManager`. +(Contributed by Brett Cannon in :issue:`25609`.) + + datetime -------- @@ -246,6 +258,14 @@ telnetlib Stéphane Wirtel in :issue:`25485`). +typing +------ + +The :class:`typing.ContextManager` class has been added for +representing :class:`contextlib.AbstractContextManager`. +(Contributed by Brett Cannon in :issue:`25609`.) + + unittest.mock ------------- @@ -372,9 +392,9 @@ become proper keywords in Python 3.7. Deprecated Python modules, functions and methods ------------------------------------------------ -* :meth:`importlib.machinery.SourceFileLoader` and - :meth:`importlib.machinery.SourcelessFileLoader` are now deprecated. They - were the only remaining implementations of +* :meth:`importlib.machinery.SourceFileLoader.load_module` and + :meth:`importlib.machinery.SourcelessFileLoader.load_module` are now + deprecated. They were the only remaining implementations of :meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 5377987..98903bc 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,11 +1,34 @@ """Utilities for with-statement contexts. See PEP 343.""" - +import abc import sys from collections import deque from functools import wraps -__all__ = ["contextmanager", "closing", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress"] +__all__ = ["contextmanager", "closing", "AbstractContextManager", + "ContextDecorator", "ExitStack", "redirect_stdout", + "redirect_stderr", "suppress"] + + +class AbstractContextManager(abc.ABC): + + """An abstract base class for context managers.""" + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractContextManager: + if (any("__enter__" in B.__dict__ for B in C.__mro__) and + any("__exit__" in B.__dict__ for B in C.__mro__)): + return True + return NotImplemented class ContextDecorator(object): @@ -31,7 +54,7 @@ class ContextDecorator(object): return inner -class _GeneratorContextManager(ContextDecorator): +class _GeneratorContextManager(ContextDecorator, AbstractContextManager): """Helper for @contextmanager decorator.""" def __init__(self, func, args, kwds): @@ -134,7 +157,7 @@ def contextmanager(func): return helper -class closing(object): +class closing(AbstractContextManager): """Context to automatically close something at the end of a block. Code like this: @@ -159,7 +182,7 @@ class closing(object): self.thing.close() -class _RedirectStream: +class _RedirectStream(AbstractContextManager): _stream = None @@ -199,7 +222,7 @@ class redirect_stderr(_RedirectStream): _stream = "stderr" -class suppress: +class suppress(AbstractContextManager): """Context manager to suppress specified exceptions After the exception is suppressed, execution proceeds with the next @@ -230,7 +253,7 @@ class suppress: # Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(object): +class ExitStack(AbstractContextManager): """Context manager for dynamic management of a stack of exit callbacks For example: @@ -309,9 +332,6 @@ class ExitStack(object): """Immediately unwind the context stack""" self.__exit__(None, None, None) - def __enter__(self): - return self - def __exit__(self, *exc_details): received_exc = exc_details[0] is not None diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 04fc875..5c8bc98 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -12,6 +12,39 @@ except ImportError: threading = None +class TestAbstractContextManager(unittest.TestCase): + + def test_enter(self): + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + manager = DefaultEnter() + self.assertIs(manager.__enter__(), manager) + + def test_exit_is_abstract(self): + class MissingExit(AbstractContextManager): + pass + + with self.assertRaises(TypeError): + MissingExit() + + def test_structural_subclassing(self): + class ManagerFromScratch: + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): + return None + + self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + + class DefaultEnter(AbstractContextManager): + def __exit__(self, *args): + super().__exit__(*args) + + self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + + class ContextManagerTestCase(unittest.TestCase): def test_contextmanager_plain(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index f1c6e12..a360824 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1,3 +1,4 @@ +import contextlib import pickle import re import sys @@ -1309,6 +1310,21 @@ class CollectionsAbcTests(TestCase): assert len(MMB[KT, VT]()) == 0 +class OtherABCTests(TestCase): + + @skipUnless(hasattr(typing, 'ContextManager'), + 'requires typing.ContextManager') + def test_contextmanager(self): + @contextlib.contextmanager + def manager(): + yield 42 + + cm = manager() + assert isinstance(cm, typing.ContextManager) + assert isinstance(cm, typing.ContextManager[int]) + assert not isinstance(42, typing.ContextManager) + + class NamedTupleTests(TestCase): def test_basics(self): @@ -1447,6 +1463,8 @@ class AllTests(TestCase): assert 'ValuesView' in a assert 'cast' in a assert 'overload' in a + if hasattr(contextlib, 'AbstractContextManager'): + assert 'ContextManager' in a # Check that io and re are not exported. assert 'io' not in a assert 're' not in a diff --git a/Lib/typing.py b/Lib/typing.py index 6ead3c4..42a9ea3 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1,6 +1,7 @@ import abc from abc import abstractmethod, abstractproperty import collections +import contextlib import functools import re as stdlib_re # Avoid confusion with the re we export. import sys @@ -1530,6 +1531,12 @@ class ValuesView(MappingView[VT_co], extra=collections_abc.ValuesView): pass +if hasattr(contextlib, 'AbstractContextManager'): + class ContextManager(Generic[T_co], extra=contextlib.AbstractContextManager): + __slots__ = () + __all__.append('ContextManager') + + class Dict(dict, MutableMapping[KT, VT]): def __new__(cls, *args, **kwds): diff --git a/Misc/NEWS b/Misc/NEWS index 9970636..56b1501 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #25609: Introduce contextlib.AbstractContextManager and + typing.ContextManager. + - Issue #26709: Fixed Y2038 problem in loading binary PLists. - Issue #23735: Handle terminal resizing with Readline 6.3+ by installing our -- cgit v0.12 From 8bd092b50115560a777ad4f9fc656e5ef8c8c89e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:16:16 -0700 Subject: Normalize whitespace --- Lib/contextlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 98903bc..6cf112a 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -27,7 +27,7 @@ class AbstractContextManager(abc.ABC): if cls is AbstractContextManager: if (any("__enter__" in B.__dict__ for B in C.__mro__) and any("__exit__" in B.__dict__ for B in C.__mro__)): - return True + return True return NotImplemented -- cgit v0.12 From ef0138f42153d467edf8b0429c531f5204fe45dc Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 12:29:05 -0700 Subject: Issue #26668: Remove the redundant Lib/test/test_importlib/regrtest.py --- Lib/test/test_importlib/regrtest.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 Lib/test/test_importlib/regrtest.py diff --git a/Lib/test/test_importlib/regrtest.py b/Lib/test/test_importlib/regrtest.py deleted file mode 100644 index 98c815c..0000000 --- a/Lib/test/test_importlib/regrtest.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Run Python's standard test suite using importlib.__import__. - -Tests known to fail because of assumptions that importlib (properly) -invalidates are automatically skipped if the entire test suite is run. -Otherwise all command-line options valid for test.regrtest are also valid for -this script. - -""" -import importlib -import sys -from test import libregrtest - -if __name__ == '__main__': - __builtins__.__import__ = importlib.__import__ - sys.path_importer_cache.clear() - - libregrtest.main(quiet=True, verbose2=True) -- cgit v0.12 From 5f0507d8ab2fee70139ff147c5f7d472e8f4ba2b Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 8 Apr 2016 15:04:28 -0700 Subject: Issue #26587: Allow .pth files to specify file paths as well as directories. Thanks to Wolfgang Langner for the bug report and initial version of the patch. --- Doc/whatsnew/3.6.rst | 8 ++++++++ Lib/site.py | 18 +++++++++--------- Lib/test/test_site.py | 2 +- Misc/ACKS | 1 + Misc/NEWS | 3 +++ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 555814e..d217c4d 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -251,6 +251,14 @@ Previously, names of properties and slots which were not yet created on an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +site +---- + +When specifying paths to add to :attr:`sys.path` in a `.pth` file, +you may now specify file paths on top of directories (e.g. zip files). +(Contributed by Wolfgang Langner in :issue:`26587`). + + telnetlib --------- diff --git a/Lib/site.py b/Lib/site.py index 56ba709..b66123f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -131,13 +131,13 @@ def removeduppaths(): def _init_pathinfo(): - """Return a set containing all existing directory entries from sys.path""" + """Return a set containing all existing file system items from sys.path.""" d = set() - for dir in sys.path: + for item in sys.path: try: - if os.path.isdir(dir): - dir, dircase = makepath(dir) - d.add(dircase) + if os.path.exists(item): + _, itemcase = makepath(item) + d.add(itemcase) except TypeError: continue return d @@ -150,9 +150,9 @@ def addpackage(sitedir, name, known_paths): """ if known_paths is None: known_paths = _init_pathinfo() - reset = 1 + reset = True else: - reset = 0 + reset = False fullname = os.path.join(sitedir, name) try: f = open(fullname, "r") @@ -190,9 +190,9 @@ def addsitedir(sitedir, known_paths=None): 'sitedir'""" if known_paths is None: known_paths = _init_pathinfo() - reset = 1 + reset = True else: - reset = 0 + reset = False sitedir, sitedircase = makepath(sitedir) if not sitedircase in known_paths: sys.path.append(sitedir) # Add path component diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 21628a9..f698927 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -75,7 +75,7 @@ class HelperFunctionsTests(unittest.TestCase): def test_init_pathinfo(self): dir_set = site._init_pathinfo() for entry in [site.makepath(path)[1] for path in sys.path - if path and os.path.isdir(path)]: + if path and os.path.exists(path)]: self.assertIn(entry, dir_set, "%s from sys.path not found in set returned " "by _init_pathinfo(): %s" % (entry, dir_set)) diff --git a/Misc/ACKS b/Misc/ACKS index 20e76f5..6dd30f3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -815,6 +815,7 @@ Torsten Landschoff Tino Lange Glenn Langford Andrew Langmead +Wolfgang Langner Detlef Lannert Soren Larsen Amos Latteier diff --git a/Misc/NEWS b/Misc/NEWS index 56b1501..6fbbdb3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -237,6 +237,9 @@ Core and Builtins Library ------- +- Issue #26587: the site module now allows .pth files to specify files to be + added to sys.path (e.g. zip files). + - Issue #25609: Introduce contextlib.AbstractContextManager and typing.ContextManager. -- cgit v0.12 From fe21de98369bf99acf4745cdf539c9c952980c92 Mon Sep 17 00:00:00 2001 From: Berker Peksag Date: Sat, 9 Apr 2016 07:34:39 +0300 Subject: Issue #26687: Use Py_RETURN_NONE macro in sqlite3 module --- Modules/_sqlite/cache.c | 3 +-- Modules/_sqlite/connection.c | 24 ++++++++---------------- Modules/_sqlite/cursor.c | 12 ++++-------- Modules/_sqlite/module.c | 9 +++------ 4 files changed, 16 insertions(+), 32 deletions(-) diff --git a/Modules/_sqlite/cache.c b/Modules/_sqlite/cache.c index 3689a4e..62c5893 100644 --- a/Modules/_sqlite/cache.c +++ b/Modules/_sqlite/cache.c @@ -244,8 +244,7 @@ PyObject* pysqlite_cache_display(pysqlite_Cache* self, PyObject* args) ptr = ptr->next; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyMethodDef cache_methods[] = { diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 22298d9..f634877 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -348,8 +348,7 @@ PyObject* pysqlite_connection_close(pysqlite_Connection* self, PyObject* args) } } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } /* @@ -857,8 +856,7 @@ PyObject* pysqlite_connection_create_function(pysqlite_Connection* self, PyObjec if (PyDict_SetItem(self->function_pinboard, func, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -889,8 +887,7 @@ PyObject* pysqlite_connection_create_aggregate(pysqlite_Connection* self, PyObje if (PyDict_SetItem(self->function_pinboard, aggregate_class, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1025,8 +1022,7 @@ static PyObject* pysqlite_connection_set_authorizer(pysqlite_Connection* self, P if (PyDict_SetItem(self->function_pinboard, authorizer_cb, Py_None) == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1055,8 +1051,7 @@ static PyObject* pysqlite_connection_set_progress_handler(pysqlite_Connection* s return NULL; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) @@ -1083,8 +1078,7 @@ static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* sel sqlite3_trace(self->db, _trace_callback, trace_callback); } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } #ifdef HAVE_LOAD_EXTENSION @@ -1107,8 +1101,7 @@ static PyObject* pysqlite_enable_load_extension(pysqlite_Connection* self, PyObj PyErr_SetString(pysqlite_OperationalError, "Error enabling load extension"); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -1131,8 +1124,7 @@ static PyObject* pysqlite_load_extension(pysqlite_Connection* self, PyObject* ar PyErr_SetString(pysqlite_OperationalError, errmsg); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } #endif diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 63d705e..552c8bb 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -242,8 +242,7 @@ PyObject* _pysqlite_build_column_name(const char* colname) const char* pos; if (!colname) { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } for (pos = colname;; pos++) { @@ -914,8 +913,7 @@ PyObject* pysqlite_cursor_fetchone(pysqlite_Cursor* self, PyObject* args) row = pysqlite_cursor_iternext(self); if (!row && !PyErr_Occurred()) { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } return row; @@ -996,8 +994,7 @@ PyObject* pysqlite_cursor_fetchall(pysqlite_Cursor* self, PyObject* args) PyObject* pysqlite_noop(pysqlite_Connection* self, PyObject* args) { /* don't care, return None */ - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyObject* pysqlite_cursor_close(pysqlite_Cursor* self, PyObject* args) @@ -1013,8 +1010,7 @@ PyObject* pysqlite_cursor_close(pysqlite_Cursor* self, PyObject* args) self->closed = 1; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } static PyMethodDef cursor_methods[] = { diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index ff2e3a5..7cd6d2a 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -139,8 +139,7 @@ static PyObject* module_enable_shared_cache(PyObject* self, PyObject* args, PyOb PyErr_SetString(pysqlite_OperationalError, "Changing the shared_cache flag failed"); return NULL; } else { - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } } @@ -172,8 +171,7 @@ static PyObject* module_register_adapter(PyObject* self, PyObject* args) if (rc == -1) return NULL; - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyDoc_STRVAR(module_register_adapter_doc, @@ -221,8 +219,7 @@ static PyObject* enable_callback_tracebacks(PyObject* self, PyObject* args) return NULL; } - Py_INCREF(Py_None); - return Py_None; + Py_RETURN_NONE; } PyDoc_STRVAR(enable_callback_tracebacks_doc, -- cgit v0.12 From 3872d62133f03a6c8292ee2f97999597f90f9082 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Sun, 10 Apr 2016 02:41:25 +0000 Subject: =?UTF-8?q?Issue=20#25609:=20Double=20back-ticks=20to=20avoid=20?= =?UTF-8?q?=E2=80=9Cmake=20check=E2=80=9D=20buildbot=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d217c4d..9be1a9c 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -196,7 +196,7 @@ contextlib The :class:`contextlib.AbstractContextManager` class has been added to provide an abstract base class for context managers. It provides a sensible default implementation for `__enter__()` which returns -`self` and leaves `__exit__()` an abstract method. A matching +``self`` and leaves `__exit__()` an abstract method. A matching class has been added to the :mod:`typing` module as :class:`typing.ContextManager`. (Contributed by Brett Cannon in :issue:`25609`.) -- cgit v0.12 From 03f17f86717372ca010273dc8946fd19914a534b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Apr 2016 14:44:59 +0300 Subject: Issue #17339: Improved TypeError message in bytes constructor. --- Objects/bytesobject.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 639ee71..cbf8166 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3469,31 +3469,24 @@ _PyBytes_FromTuple(PyObject *x) } static PyObject * -_PyBytes_FromIterator(PyObject *x) +_PyBytes_FromIterator(PyObject *it, PyObject *x) { char *str; - PyObject *it; Py_ssize_t i, size; _PyBytesWriter writer; - _PyBytesWriter_Init(&writer); - /* For iterator version, create a string object and resize as needed */ size = PyObject_LengthHint(x, 64); if (size == -1 && PyErr_Occurred()) return NULL; + _PyBytesWriter_Init(&writer); str = _PyBytesWriter_Alloc(&writer, size); if (str == NULL) return NULL; writer.overallocate = 1; size = writer.allocated; - /* Get the iterator */ - it = PyObject_GetIter(x); - if (it == NULL) - goto error; - /* Run the iterator to exhaustion */ for (i = 0; ; i++) { PyObject *item; @@ -3529,19 +3522,19 @@ _PyBytes_FromIterator(PyObject *x) } *str++ = (char) value; } - Py_DECREF(it); return _PyBytesWriter_Finish(&writer, str); error: _PyBytesWriter_Dealloc(&writer); - Py_XDECREF(it); return NULL; } PyObject * PyBytes_FromObject(PyObject *x) { + PyObject *it, *result; + if (x == NULL) { PyErr_BadInternalCall(); return NULL; @@ -3562,13 +3555,19 @@ PyBytes_FromObject(PyObject *x) if (PyTuple_CheckExact(x)) return _PyBytes_FromTuple(x); - if (PyUnicode_Check(x)) { - PyErr_SetString(PyExc_TypeError, - "cannot convert unicode object to bytes"); - return NULL; + if (!PyUnicode_Check(x)) { + it = PyObject_GetIter(x); + if (it != NULL) { + result = _PyBytes_FromIterator(it, x); + Py_DECREF(it); + return result; + } } - return _PyBytes_FromIterator(x); + PyErr_Format(PyExc_TypeError, + "cannot convert '%.200s' object to bytes", + x->ob_type->tp_name); + return NULL; } static PyObject * -- cgit v0.12 From 47c5474aa0cbe8dc3cf2c370b19769edd3f2e8d8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 10 Apr 2016 15:46:30 +0300 Subject: Issue #26623: TypeError message for JSON unserializible object now contains object's type name, not object's representation. Based on patch by Mahmoud Lababidi. --- Lib/json/encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index d596489..0772bbc 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,7 +176,8 @@ class JSONEncoder(object): return JSONEncoder.default(self, o) """ - raise TypeError(repr(o) + " is not JSON serializable") + raise TypeError("Object of type '%s' is not JSON serializable" % + o.__class__.__name__) def encode(self, o): """Return a JSON string representation of a Python data structure. -- cgit v0.12 From 50ab1a3694c43b9ab6798b98d9e5983c78cb17e2 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 11 Apr 2016 00:38:12 +0000 Subject: Issue #26685: Raise OSError if closing a socket fails --- Doc/library/socket.rst | 4 ++++ Doc/whatsnew/3.6.rst | 4 ++++ Lib/test/test_pty.py | 1 - Lib/test/test_socket.py | 11 +++++++++++ Misc/NEWS | 2 ++ Modules/socketmodule.c | 6 +++++- 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c09927c..cd6e310 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -868,6 +868,10 @@ to sockets. it is recommended to :meth:`close` them explicitly, or to use a :keyword:`with` statement around them. + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`close` call is made. + .. note:: :meth:`close()` releases the resource associated with a connection but diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9be1a9c..8bf2847 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -514,6 +514,10 @@ Changes in the Python API * :func:`spwd.getspnam` now raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user doesn't have privileges. +* The :meth:`socket.socket.close` method now raises an exception if + an error (e.g. EBADF) was reported by the underlying system call. + See :issue:`26685`. + Changes in the C API -------------------- diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index ef5e99e..15f88e4 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -277,7 +277,6 @@ class SmallPtyTests(unittest.TestCase): socketpair = self._socketpair() masters = [s.fileno() for s in socketpair] - os.close(masters[1]) socketpair[1].close() os.close(write_to_stdin_fd) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 02bc0c0..982a976 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -1161,6 +1161,17 @@ class GeneralModuleTests(unittest.TestCase): sock.close() self.assertRaises(OSError, sock.send, b"spam") + def testCloseException(self): + sock = socket.socket() + socket.socket(fileno=sock.fileno()).close() + try: + sock.close() + except OSError as err: + # Winsock apparently raises ENOTSOCK + self.assertIn(err.errno, (errno.EBADF, errno.ENOTSOCK)) + else: + self.fail("close() should raise EBADF/ENOTSOCK") + def testNewAttributes(self): # testing .family, .type and .protocol diff --git a/Misc/NEWS b/Misc/NEWS index f7694cc..7ac9b34 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,8 @@ Core and Builtins Library ------- +- Issue #26685: Raise OSError if closing a socket fails. + - Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. - Issue #13952: Add .csv to mimetypes.types_map. Patch by Geoff Wilson. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8df735d..bcff004 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2576,6 +2576,7 @@ static PyObject * sock_close(PySocketSockObject *s) { SOCKET_T fd; + int res; fd = s->sock_fd; if (fd != -1) { @@ -2586,8 +2587,11 @@ sock_close(PySocketSockObject *s) http://linux.derkeiler.com/Mailing-Lists/Kernel/2005-09/3000.html for more details. */ Py_BEGIN_ALLOW_THREADS - (void) SOCKETCLOSE(fd); + res = SOCKETCLOSE(fd); Py_END_ALLOW_THREADS + if (res < 0) { + return s->errorhandler(); + } } Py_INCREF(Py_None); return Py_None; -- cgit v0.12 From da3bb38452740cec27723d7dc89a926547613204 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Mon, 11 Apr 2016 00:40:08 +0000 Subject: Issue #26585: Eliminate _quote_html() and use html.escape(quote=False) Patch by Xiang Zhang. --- Lib/http/server.py | 16 ++++++++-------- Lib/test/test_httpservers.py | 30 +++++++++++++++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/Lib/http/server.py b/Lib/http/server.py index f4ad260..fbee6a9 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -127,9 +127,6 @@ DEFAULT_ERROR_MESSAGE = """\ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" -def _quote_html(html): - return html.replace("&", "&").replace("<", "<").replace(">", ">") - class HTTPServer(socketserver.TCPServer): allow_reuse_address = 1 # Seems to make sense in testing environment @@ -449,9 +446,12 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): if explain is None: explain = longmsg self.log_error("code %d, message %s", code, message) - # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) - content = (self.error_message_format % - {'code': code, 'message': _quote_html(message), 'explain': _quote_html(explain)}) + # HTML encode to prevent Cross Site Scripting attacks (see bug #1100201) + content = (self.error_message_format % { + 'code': code, + 'message': html.escape(message, quote=False), + 'explain': html.escape(explain, quote=False) + }) body = content.encode('UTF-8', 'replace') self.send_response(code, message) self.send_header("Content-Type", self.error_content_type) @@ -710,7 +710,7 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): errors='surrogatepass') except UnicodeDecodeError: displaypath = urllib.parse.unquote(path) - displaypath = html.escape(displaypath) + displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = 'Directory listing for %s' % displaypath r.append('%s' % (urllib.parse.quote(linkname, errors='surrogatepass'), - html.escape(displayname))) + html.escape(displayname, quote=False))) r.append('\n
\n\n\n') encoded = '\n'.join(r).encode(enc, 'surrogateescape') f = io.BytesIO() diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index c752fd8..3856d00 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -344,7 +344,7 @@ class SimpleHTTPServerTestCase(BaseTestCase): quotedname = urllib.parse.quote(filename, errors='surrogatepass') self.assertIn(('href="%s"' % quotedname) .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename)) + self.assertIn(('>%s<' % html.escape(filename, quote=False)) .encode(enc, 'surrogateescape'), body) response = self.request(self.base_url + '/' + quotedname) self.check_status_and_reason(response, HTTPStatus.OK, @@ -422,6 +422,27 @@ class SimpleHTTPServerTestCase(BaseTestCase): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") + def test_html_escape_filename(self): + filename = '.txt' + fullpath = os.path.join(self.tempdir, filename) + + try: + open(fullpath, 'w').close() + except OSError: + raise unittest.SkipTest('Can not create file %s on current file ' + 'system' % filename) + + try: + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + enc = response.headers.get_content_charset() + finally: + os.unlink(fullpath) # avoid affecting test_undecodable_filename + + self.assertIsNotNone(enc) + html_text = '>%s<' % html.escape(filename, quote=False) + self.assertIn(html_text.encode(enc), body) + cgi_file1 = """\ #!%s @@ -883,6 +904,13 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.assertFalse(self.handler.get_called) self.assertEqual(self.handler.requestline, 'GET / HTTP/1.1') + def test_html_escape_on_error(self): + result = self.send_typical_request( + b' / HTTP/1.1') + result = b''.join(result) + text = '' + self.assertIn(html.escape(text, quote=False).encode('ascii'), result) + def test_close_connection(self): # handle_one_request() should be repeatedly called until # it sets close_connection diff --git a/Misc/NEWS b/Misc/NEWS index 7ac9b34..7284d3f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,9 @@ Core and Builtins Library ------- +- Issue #26585: Eliminate http.server._quote_html() and use + html.escape(quote=False). Patch by Xiang Zhang. + - Issue #26685: Raise OSError if closing a socket fails. - Issue #16329: Add .webm to mimetypes.types_map. Patch by Giampaolo Rodola'. -- cgit v0.12 From 7b228231c78f49986928cd587cbf85197a3b3987 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:15:26 +0200 Subject: Issue #26647: Cleanup opcode Simplify code to build opcode.opname. Patch written by Demur Rumed. --- Lib/opcode.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 71ce672..ecc24bb 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -34,9 +34,7 @@ hasfree = [] hasnargs = [] opmap = {} -opname = [''] * 256 -for op in range(256): opname[op] = '<%r>' % (op,) -del op +opname = ['<%r>' % (op,) for op in range(256)] def def_op(name, op): opname[op] = name -- cgit v0.12 From 7e50978a4e36cd4ab4b363ee0c77abefc13e2c23 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:17:06 +0200 Subject: Issue #26647: Cleanup modulefinder Use directly dis.opmap[name] rather than dis.opname.index(name). Patch written by Demur Rumed. --- Lib/modulefinder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 4a2f1b5..d8d645c 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -14,11 +14,11 @@ with warnings.catch_warnings(): import imp # XXX Clean up once str8's cstor matches bytes. -LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')]) -IMPORT_NAME = bytes([dis.opname.index('IMPORT_NAME')]) -STORE_NAME = bytes([dis.opname.index('STORE_NAME')]) -STORE_GLOBAL = bytes([dis.opname.index('STORE_GLOBAL')]) -STORE_OPS = [STORE_NAME, STORE_GLOBAL] +LOAD_CONST = bytes([dis.opmap['LOAD_CONST']]) +IMPORT_NAME = bytes([dis.opmap['IMPORT_NAME']]) +STORE_NAME = bytes([dis.opmap['STORE_NAME']]) +STORE_GLOBAL = bytes([dis.opmap['STORE_GLOBAL']]) +STORE_OPS = STORE_NAME, STORE_GLOBAL HAVE_ARGUMENT = bytes([dis.HAVE_ARGUMENT]) # Modulefinder does a good job at simulating Python's, but it can not -- cgit v0.12 From 328cb1fed0c91f50f311cdc545fe0e9303d0dae7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 12 Apr 2016 18:46:10 +0200 Subject: Update pygettext.py to get ride of imp Issue #26639: Replace imp with importlib in Tools/i18n/pygettext.py. Remove _get_modpkg_path(), replaced with importlib.util.find_spec(). --- Tools/i18n/pygettext.py | 51 ++++++++----------------------------------------- 1 file changed, 8 insertions(+), 43 deletions(-) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 3c6c14c..be941c7 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -156,7 +156,8 @@ If `inputfile' is -, standard input is read. """) import os -import imp +import importlib.machinery +import importlib.util import sys import glob import time @@ -263,8 +264,7 @@ def _visit_pyfiles(list, dirname, names): # get extension for python source files if '_py_ext' not in globals(): global _py_ext - _py_ext = [triple[0] for triple in imp.get_suffixes() - if triple[2] == imp.PY_SOURCE][0] + _py_ext = importlib.machinery.SOURCE_SUFFIXES[0] # don't recurse into CVS directories if 'CVS' in names: @@ -277,45 +277,6 @@ def _visit_pyfiles(list, dirname, names): ) -def _get_modpkg_path(dotted_name, pathlist=None): - """Get the filesystem path for a module or a package. - - Return the file system path to a file for a module, and to a directory for - a package. Return None if the name is not found, or is a builtin or - extension module. - """ - # split off top-most name - parts = dotted_name.split('.', 1) - - if len(parts) > 1: - # we have a dotted path, import top-level package - try: - file, pathname, description = imp.find_module(parts[0], pathlist) - if file: file.close() - except ImportError: - return None - - # check if it's indeed a package - if description[2] == imp.PKG_DIRECTORY: - # recursively handle the remaining name parts - pathname = _get_modpkg_path(parts[1], [pathname]) - else: - pathname = None - else: - # plain name - try: - file, pathname, description = imp.find_module( - dotted_name, pathlist) - if file: - file.close() - if description[2] not in [imp.PY_SOURCE, imp.PKG_DIRECTORY]: - pathname = None - except ImportError: - pathname = None - - return pathname - - def getFilesForName(name): """Get a list of module files for a filename, a module or package name, or a directory. @@ -330,7 +291,11 @@ def getFilesForName(name): return list # try to find module or package - name = _get_modpkg_path(name) + try: + spec = importlib.util.find_spec(name) + name = spec.origin + except ImportError: + name = None if not name: return [] -- cgit v0.12 From 0cab9c1ebaa11bb7838a552c671c903156262ab7 Mon Sep 17 00:00:00 2001 From: Martin Panter Date: Wed, 13 Apr 2016 00:36:52 +0000 Subject: Issue #26404: Add context manager to socketserver, by Aviv Palivoda --- Doc/library/http.server.rst | 7 +++-- Doc/library/socketserver.rst | 55 ++++++++++++++++++++++------------------ Doc/library/wsgiref.rst | 32 +++++++++++------------ Doc/library/xmlrpc.server.rst | 59 +++++++++++++++++++++---------------------- Doc/whatsnew/3.6.rst | 10 ++++++++ Lib/http/server.py | 18 ++++++------- Lib/socketserver.py | 6 +++++ Lib/test/test_socketserver.py | 7 ++++- Lib/wsgiref/simple_server.py | 13 +++++----- Lib/xmlrpc/server.py | 25 +++++++++--------- Misc/NEWS | 2 ++ 11 files changed, 128 insertions(+), 106 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 0bde35b..7ab249a 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -375,10 +375,9 @@ the current directory:: Handler = http.server.SimpleHTTPRequestHandler - httpd = socketserver.TCPServer(("", PORT), Handler) - - print("serving at port", PORT) - httpd.serve_forever() + with socketserver.TCPServer(("", PORT), Handler) as httpd: + print("serving at port", PORT) + httpd.serve_forever() .. _http-server-cli: diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index aaaa61e..e148d30 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -52,11 +52,12 @@ handler class by subclassing the :class:`BaseRequestHandler` class and overriding its :meth:`~BaseRequestHandler.handle` method; this method will process incoming requests. Second, you must instantiate one of the server classes, passing it -the server's address and the request handler class. Then call the +the server's address and the request handler class. It is recommended to use +the server in a :keyword:`with` statement. Then call the :meth:`~BaseServer.handle_request` or :meth:`~BaseServer.serve_forever` method of the server object to process one or many requests. Finally, call :meth:`~BaseServer.server_close` -to close the socket. +to close the socket (unless you used a :keyword:`with` statement). When inheriting from :class:`ThreadingMixIn` for threaded connection behavior, you should explicitly declare how you want your threads to behave on an abrupt @@ -353,6 +354,11 @@ Server Objects default implementation always returns :const:`True`. + .. versionchanged:: 3.6 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`server_close`. + + Request Handler Objects ----------------------- @@ -433,11 +439,10 @@ This is the server side:: HOST, PORT = "localhost", 9999 # Create the server, binding to localhost on port 9999 - server = socketserver.TCPServer((HOST, PORT), MyTCPHandler) - - # Activate the server; this will keep running until you - # interrupt the program with Ctrl-C - server.serve_forever() + with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: + # Activate the server; this will keep running until you + # interrupt the program with Ctrl-C + server.serve_forever() An alternative request handler class that makes use of streams (file-like objects that simplify communication by providing the standard file interface):: @@ -529,8 +534,8 @@ This is the server side:: if __name__ == "__main__": HOST, PORT = "localhost", 9999 - server = socketserver.UDPServer((HOST, PORT), MyUDPHandler) - server.serve_forever() + with socketserver.UDPServer((HOST, PORT), MyUDPHandler) as server: + server.serve_forever() This is the client side:: @@ -592,22 +597,22 @@ An example for the :class:`ThreadingMixIn` class:: HOST, PORT = "localhost", 0 server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler) - ip, port = server.server_address - - # Start a thread with the server -- that thread will then start one - # more thread for each request - server_thread = threading.Thread(target=server.serve_forever) - # Exit the server thread when the main thread terminates - server_thread.daemon = True - server_thread.start() - print("Server loop running in thread:", server_thread.name) - - client(ip, port, "Hello World 1") - client(ip, port, "Hello World 2") - client(ip, port, "Hello World 3") - - server.shutdown() - server.server_close() + with server: + ip, port = server.server_address + + # Start a thread with the server -- that thread will then start one + # more thread for each request + server_thread = threading.Thread(target=server.serve_forever) + # Exit the server thread when the main thread terminates + server_thread.daemon = True + server_thread.start() + print("Server loop running in thread:", server_thread.name) + + client(ip, port, "Hello World 1") + client(ip, port, "Hello World 2") + client(ip, port, "Hello World 3") + + server.shutdown() The output of the example should look something like this:: diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 71607d6..8d62885 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -131,9 +131,9 @@ parameter expect a WSGI-compliant dictionary to be supplied; please see for key, value in environ.items()] return ret - httpd = make_server('', 8000, simple_app) - print("Serving on port 8000...") - httpd.serve_forever() + with make_server('', 8000, simple_app) as httpd: + print("Serving on port 8000...") + httpd.serve_forever() In addition to the environment functions above, the :mod:`wsgiref.util` module @@ -283,14 +283,14 @@ request. (E.g., using the :func:`shift_path_info` function from from wsgiref.simple_server import make_server, demo_app - httpd = make_server('', 8000, demo_app) - print("Serving HTTP on port 8000...") + with make_server('', 8000, demo_app) as httpd: + print("Serving HTTP on port 8000...") - # Respond to requests until process is killed - httpd.serve_forever() + # Respond to requests until process is killed + httpd.serve_forever() - # Alternative: serve one request, then exit - httpd.handle_request() + # Alternative: serve one request, then exit + httpd.handle_request() .. function:: demo_app(environ, start_response) @@ -430,9 +430,9 @@ Paste" library. # This is the application wrapped in a validator validator_app = validator(simple_app) - httpd = make_server('', 8000, validator_app) - print("Listening on port 8000....") - httpd.serve_forever() + with make_server('', 8000, validator_app) as httpd: + print("Listening on port 8000....") + httpd.serve_forever() :mod:`wsgiref.handlers` -- server/gateway base classes @@ -769,8 +769,8 @@ This is a working "Hello World" WSGI application:: # The returned object is going to be printed return [b"Hello World"] - httpd = make_server('', 8000, hello_world_app) - print("Serving on port 8000...") + with make_server('', 8000, hello_world_app) as httpd: + print("Serving on port 8000...") - # Serve until process is killed - httpd.serve_forever() + # Serve until process is killed + httpd.serve_forever() diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 680db41..ca80aab 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -147,29 +147,29 @@ Server code:: rpc_paths = ('/RPC2',) # Create server - server = SimpleXMLRPCServer(("localhost", 8000), - requestHandler=RequestHandler) - server.register_introspection_functions() + with SimpleXMLRPCServer(("localhost", 8000), + requestHandler=RequestHandler) as server: + server.register_introspection_functions() - # Register pow() function; this will use the value of - # pow.__name__ as the name, which is just 'pow'. - server.register_function(pow) + # Register pow() function; this will use the value of + # pow.__name__ as the name, which is just 'pow'. + server.register_function(pow) - # Register a function under a different name - def adder_function(x,y): - return x + y - server.register_function(adder_function, 'add') + # Register a function under a different name + def adder_function(x,y): + return x + y + server.register_function(adder_function, 'add') - # Register an instance; all the methods of the instance are - # published as XML-RPC methods (in this case, just 'mul'). - class MyFuncs: - def mul(self, x, y): - return x * y + # Register an instance; all the methods of the instance are + # published as XML-RPC methods (in this case, just 'mul'). + class MyFuncs: + def mul(self, x, y): + return x * y - server.register_instance(MyFuncs()) + server.register_instance(MyFuncs()) - # Run the server's main loop - server.serve_forever() + # Run the server's main loop + server.serve_forever() The following client code will call the methods made available by the preceding server:: @@ -206,18 +206,17 @@ a server allowing dotted names and registering a multicall function. def getCurrentTime(): return datetime.datetime.now() - server = SimpleXMLRPCServer(("localhost", 8000)) - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - server.server_close() - sys.exit(0) + with SimpleXMLRPCServer(("localhost", 8000)) as server: + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_instance(ExampleService(), allow_dotted_names=True) + server.register_multicall_functions() + print('Serving XML-RPC on localhost port 8000') + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) This ExampleService demo can be invoked from the command line:: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 8bf2847..ef10ef2 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -259,6 +259,16 @@ you may now specify file paths on top of directories (e.g. zip files). (Contributed by Wolfgang Langner in :issue:`26587`). +socketserver +------------ + +Servers based on the :mod:`socketserver` module, including those +defined in :mod:`http.server`, :mod:`xmlrpc.server` and +:mod:`wsgiref.simple_server`, now support the :term:`context manager` +protocol. +(Contributed by Aviv Palivoda in :issue:`26404`.) + + telnetlib --------- diff --git a/Lib/http/server.py b/Lib/http/server.py index fbee6a9..c1607b3 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1175,16 +1175,14 @@ def test(HandlerClass=BaseHTTPRequestHandler, server_address = (bind, port) HandlerClass.protocol_version = protocol - httpd = ServerClass(server_address, HandlerClass) - - sa = httpd.socket.getsockname() - print("Serving HTTP on", sa[0], "port", sa[1], "...") - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - httpd.server_close() - sys.exit(0) + with ServerClass(server_address, HandlerClass) as httpd: + sa = httpd.socket.getsockname() + print("Serving HTTP on", sa[0], "port", sa[1], "...") + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 2f39514..3e1f058 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -378,6 +378,12 @@ class BaseServer: traceback.print_exc() print('-'*40, file=sys.stderr) + def __enter__(self): + return self + + def __exit__(self, *args): + self.server_close() + class TCPServer(BaseServer): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 27fe01c..554c106 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -104,7 +104,6 @@ class SocketServerTest(unittest.TestCase): class MyServer(svrcls): def handle_error(self, request, client_address): self.close_request(request) - self.server_close() raise class MyHandler(hdlrbase): @@ -280,6 +279,12 @@ class SocketServerTest(unittest.TestCase): socketserver.TCPServer((HOST, -1), socketserver.StreamRequestHandler) + def test_context_manager(self): + with socketserver.TCPServer((HOST, 0), + socketserver.StreamRequestHandler) as server: + pass + self.assertEqual(-1, server.socket.fileno()) + class ErrorHandlerTest(unittest.TestCase): """Test that the servers pass normal exceptions from the handler to diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index 378b316..1807c66 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -156,10 +156,9 @@ def make_server( if __name__ == '__main__': - httpd = make_server('', 8000, demo_app) - sa = httpd.socket.getsockname() - print("Serving HTTP on", sa[0], "port", sa[1], "...") - import webbrowser - webbrowser.open('http://localhost:8000/xyz?abc') - httpd.handle_request() # serve one request, then exit - httpd.server_close() + with make_server('', 8000, demo_app) as httpd: + sa = httpd.socket.getsockname() + print("Serving HTTP on", sa[0], "port", sa[1], "...") + import webbrowser + webbrowser.open('http://localhost:8000/xyz?abc') + httpd.handle_request() # serve one request, then exit diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 5b5bf7c..78728f2 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -971,16 +971,15 @@ if __name__ == '__main__': def getCurrentTime(): return datetime.datetime.now() - server = SimpleXMLRPCServer(("localhost", 8000)) - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - print('It is advisable to run this example server within a secure, closed network.') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - server.server_close() - sys.exit(0) + with SimpleXMLRPCServer(("localhost", 8000)) as server: + server.register_function(pow) + server.register_function(lambda x,y: x+y, 'add') + server.register_instance(ExampleService(), allow_dotted_names=True) + server.register_multicall_functions() + print('Serving XML-RPC on localhost port 8000') + print('It is advisable to run this example server within a secure, closed network.') + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nKeyboard interrupt received, exiting.") + sys.exit(0) diff --git a/Misc/NEWS b/Misc/NEWS index d98c184..a4109ab 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -240,6 +240,8 @@ Core and Builtins Library ------- +- Issue #26404: Add context manager to socketserver. Patch by Aviv Palivoda. + - Issue #26735: Fix :func:`os.urandom` on Solaris 11.3 and newer when reading more than 1,024 bytes: call ``getrandom()`` multiple times with a limit of 1024 bytes per call. -- cgit v0.12 From 21a663ea2829b6808dd6981904c393332d271f8e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 13 Apr 2016 15:37:23 +0300 Subject: Issue #26057: Got rid of nonneeded use of PyUnicode_FromObject(). --- Modules/_codecsmodule.c | 231 ++++--------------- Modules/socketmodule.c | 44 ++-- Objects/stringlib/find.h | 13 +- Objects/unicodeobject.c | 562 ++++++++++++++--------------------------------- Python/bltinmodule.c | 5 +- Python/getargs.c | 26 +-- 6 files changed, 240 insertions(+), 641 deletions(-) diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 7575773..1951da9 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -20,10 +20,6 @@ _decode(char_buffer_obj[,errors='strict']) -> (Unicode object, bytes consumed) - _encode() interfaces also accept non-Unicode object as - input. The objects are then converted to Unicode using - PyUnicode_FromObject() prior to applying the conversion. - These s are available: utf_8, unicode_escape, raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit), mbcs (on win32). @@ -718,7 +714,7 @@ _codecs_unicode_internal_encode_impl(PyModuleDef *module, PyObject *obj, /*[clinic input] _codecs.utf_7_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -728,22 +724,13 @@ _codecs_utf_7_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=a7accc496a32b759 input=fd91a78f103b0421]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF7(str, 0, 0, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_8_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -753,17 +740,8 @@ _codecs_utf_8_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ec831d80e7aedede input=2c22d40532f071f3]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsEncodedString(str, "utf-8", errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsUTF8String(str, errors), + PyUnicode_GET_LENGTH(str)); } /* This version provides access to the byteorder parameter of the @@ -775,7 +753,7 @@ _codecs_utf_8_encode_impl(PyModuleDef *module, PyObject *str, /*[clinic input] _codecs.utf_16_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL byteorder: int = 0 / @@ -786,22 +764,13 @@ _codecs_utf_16_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, int byteorder) /*[clinic end generated code: output=93ac58e960a9ee4d input=3935a489b2d5385e]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, byteorder), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_16_le_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -811,22 +780,13 @@ _codecs_utf_16_le_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=422bedb8da34fb66 input=bc27df05d1d20dfe]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, -1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_16_be_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -836,17 +796,8 @@ _codecs_utf_16_be_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=3aa7ee9502acdd77 input=5a69d4112763462b]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF16(str, errors, +1), + PyUnicode_GET_LENGTH(str)); } /* This version provides access to the byteorder parameter of the @@ -858,7 +809,7 @@ _codecs_utf_16_be_encode_impl(PyModuleDef *module, PyObject *str, /*[clinic input] _codecs.utf_32_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL byteorder: int = 0 / @@ -869,22 +820,13 @@ _codecs_utf_32_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, int byteorder) /*[clinic end generated code: output=3e7d5a003b02baed input=434a1efa492b8d58]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, byteorder), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_32_le_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -894,22 +836,13 @@ _codecs_utf_32_le_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=5dda641cd33dbfc2 input=dfa2d7dc78b99422]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, -1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.utf_32_be_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -919,22 +852,13 @@ _codecs_utf_32_be_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ccca8b44d91a7c7a input=4595617b18169002]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeUTF32(str, errors, +1), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.unicode_escape_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -944,22 +868,13 @@ _codecs_unicode_escape_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=389f23d2b8f8d80b input=8273506f14076912]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsUnicodeEscapeString(str), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_AsUnicodeEscapeString(str), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.raw_unicode_escape_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -969,22 +884,13 @@ _codecs_raw_unicode_escape_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=fec4e39d6ec37a62 input=181755d5dfacef3c]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_AsRawUnicodeEscapeString(str), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.latin_1_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -994,22 +900,13 @@ _codecs_latin_1_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=ecf00eb8e48c889c input=f03f6dcf1d84bee4]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_AsLatin1String(str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsLatin1String(str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.ascii_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1019,22 +916,13 @@ _codecs_ascii_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=a9d18fc6b6b91cfb input=d87e25a10a593fee]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_AsASCIIString(str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_AsASCIIString(str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.charmap_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL mapping: object = NULL / @@ -1045,20 +933,11 @@ _codecs_charmap_encode_impl(PyModuleDef *module, PyObject *str, const char *errors, PyObject *mapping) /*[clinic end generated code: output=14ca42b83853c643 input=85f4172661e8dad9]*/ { - PyObject *v; - if (mapping == Py_None) mapping = NULL; - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(_PyUnicode_EncodeCharmap(str, mapping, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(_PyUnicode_EncodeCharmap(str, mapping, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] @@ -1078,7 +957,7 @@ _codecs_charmap_build_impl(PyModuleDef *module, PyObject *map) /*[clinic input] _codecs.mbcs_encode - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1088,23 +967,14 @@ _codecs_mbcs_encode_impl(PyModuleDef *module, PyObject *str, const char *errors) /*[clinic end generated code: output=d1a013bc68798bd7 input=65c09ee1e4203263]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_EncodeCodePage(CP_ACP, str, errors), + PyUnicode_GET_LENGTH(str)); } /*[clinic input] _codecs.code_page_encode code_page: int - str: object + str: unicode errors: str(accept={str, NoneType}) = NULL / [clinic start generated code]*/ @@ -1114,19 +984,8 @@ _codecs_code_page_encode_impl(PyModuleDef *module, int code_page, PyObject *str, const char *errors) /*[clinic end generated code: output=3b406618dbfbce25 input=c8562ec460c2e309]*/ { - PyObject *v; - - str = PyUnicode_FromObject(str); - if (str == NULL || PyUnicode_READY(str) < 0) { - Py_XDECREF(str); - return NULL; - } - v = codec_tuple(PyUnicode_EncodeCodePage(code_page, - str, - errors), - PyUnicode_GET_LENGTH(str)); - Py_DECREF(str); - return v; + return codec_tuple(PyUnicode_EncodeCodePage(code_page, str, errors), + PyUnicode_GET_LENGTH(str)); } #endif /* HAVE_MBCS */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bcff004..ec35fb9 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1401,7 +1401,7 @@ static int idna_converter(PyObject *obj, struct maybe_idna *data) { size_t len; - PyObject *obj2, *obj3; + PyObject *obj2; if (obj == NULL) { idna_cleanup(data); return 1; @@ -1416,31 +1416,27 @@ idna_converter(PyObject *obj, struct maybe_idna *data) data->buf = PyByteArray_AsString(obj); len = PyByteArray_Size(obj); } - else if (PyUnicode_Check(obj) && PyUnicode_READY(obj) == 0 && PyUnicode_IS_COMPACT_ASCII(obj)) { - data->buf = PyUnicode_DATA(obj); - len = PyUnicode_GET_LENGTH(obj); - } - else { - obj2 = PyUnicode_FromObject(obj); - if (!obj2) { - PyErr_Format(PyExc_TypeError, "string or unicode text buffer expected, not %s", - obj->ob_type->tp_name); - return 0; + else if (PyUnicode_Check(obj)) { + if (PyUnicode_READY(obj) == 0 && PyUnicode_IS_COMPACT_ASCII(obj)) { + data->buf = PyUnicode_DATA(obj); + len = PyUnicode_GET_LENGTH(obj); } - obj3 = PyUnicode_AsEncodedString(obj2, "idna", NULL); - Py_DECREF(obj2); - if (!obj3) { - PyErr_SetString(PyExc_TypeError, "encoding of hostname failed"); - return 0; - } - if (!PyBytes_Check(obj3)) { - Py_DECREF(obj3); - PyErr_SetString(PyExc_TypeError, "encoding of hostname failed to return bytes"); - return 0; + else { + obj2 = PyUnicode_AsEncodedString(obj, "idna", NULL); + if (!obj2) { + PyErr_SetString(PyExc_TypeError, "encoding of hostname failed"); + return 0; + } + assert(PyBytes_Check(obj2)); + data->obj = obj2; + data->buf = PyBytes_AS_STRING(obj2); + len = PyBytes_GET_SIZE(obj2); } - data->obj = obj3; - data->buf = PyBytes_AS_STRING(obj3); - len = PyBytes_GET_SIZE(obj3); + } + else { + PyErr_Format(PyExc_TypeError, "str, bytes or bytearray expected, not %s", + obj->ob_type->tp_name); + return 0; } if (strlen(data->buf) != len) { Py_CLEAR(data->obj); diff --git a/Objects/stringlib/find.h b/Objects/stringlib/find.h index 14815f6..a7065fc 100644 --- a/Objects/stringlib/find.h +++ b/Objects/stringlib/find.h @@ -123,11 +123,6 @@ STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args, /* Wraps stringlib_parse_args_finds() and additionally ensures that the first argument is a unicode object. - -Note that we receive a pointer to the pointer of the substring object, -so when we create that object in this function we don't DECREF it, -because it continues living in the caller functions (those functions, -after finishing using the substring, must DECREF it). */ Py_LOCAL_INLINE(int) @@ -135,14 +130,10 @@ STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args, PyObject **substring, Py_ssize_t *start, Py_ssize_t *end) { - PyObject *tmp_substring; - - if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring, + if(STRINGLIB(parse_args_finds)(function_name, args, substring, start, end)) { - tmp_substring = PyUnicode_FromObject(tmp_substring); - if (!tmp_substring) + if (ensure_unicode(*substring) < 0) return 0; - *substring = tmp_substring; return 1; } return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8dc2a38..d3bb13a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -751,6 +751,18 @@ make_bloom_mask(int kind, void* ptr, Py_ssize_t len) #undef BLOOM_UPDATE } +static int +ensure_unicode(PyObject *obj) +{ + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "must be str, not %.100s", + Py_TYPE(obj)->tp_name); + return -1; + } + return PyUnicode_READY(obj); +} + /* Compilation of templated routines */ #include "stringlib/asciilib.h" @@ -3066,7 +3078,7 @@ PyUnicode_FromEncodedObject(PyObject *obj, /* Retrieve a bytes buffer view through the PEP 3118 buffer interface */ if (PyObject_GetBuffer(obj, &buffer, PyBUF_SIMPLE) < 0) { PyErr_Format(PyExc_TypeError, - "coercing to str: need a bytes-like object, %.80s found", + "decoding to str: need a bytes-like object, %.80s found", Py_TYPE(obj)->tp_name); return NULL; } @@ -3787,19 +3799,17 @@ PyUnicode_FSConverter(PyObject* arg, void* addr) output = arg; Py_INCREF(output); } - else { - arg = PyUnicode_FromObject(arg); - if (!arg) - return 0; + else if (PyUnicode_Check(arg)) { output = PyUnicode_EncodeFSDefault(arg); - Py_DECREF(arg); if (!output) return 0; - if (!PyBytes_Check(output)) { - Py_DECREF(output); - PyErr_SetString(PyExc_TypeError, "encoder failed to return bytes"); - return 0; - } + assert(PyBytes_Check(output)); + } + else { + PyErr_Format(PyExc_TypeError, + "must be str or bytes, not %.100s", + Py_TYPE(arg)->tp_name); + return 0; } size = PyBytes_GET_SIZE(output); data = PyBytes_AS_STRING(output); @@ -3871,7 +3881,7 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) if (PyUnicode_UTF8(unicode) == NULL) { assert(!PyUnicode_IS_COMPACT_ASCII(unicode)); - bytes = _PyUnicode_AsUTF8String(unicode, "strict"); + bytes = _PyUnicode_AsUTF8String(unicode, NULL); if (bytes == NULL) return NULL; _PyUnicode_UTF8(unicode) = PyObject_MALLOC(PyBytes_GET_SIZE(bytes) + 1); @@ -8860,10 +8870,8 @@ _PyUnicode_TranslateCharmap(PyObject *input, kind = PyUnicode_KIND(input); size = PyUnicode_GET_LENGTH(input); - if (size == 0) { - Py_INCREF(input); - return input; - } + if (size == 0) + return PyUnicode_FromObject(input); /* allocate enough for a simple 1:1 translation without replacements, if we need more, we'll resize */ @@ -8974,14 +8982,9 @@ PyUnicode_Translate(PyObject *str, PyObject *mapping, const char *errors) { - PyObject *result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0) return NULL; - result = _PyUnicode_TranslateCharmap(str, mapping, errors); - Py_DECREF(str); - return result; + return _PyUnicode_TranslateCharmap(str, mapping, errors); } static Py_UCS4 @@ -9163,9 +9166,10 @@ PyUnicode_EncodeDecimal(Py_UNICODE *s, } static Py_ssize_t -any_find_slice(int direction, PyObject* s1, PyObject* s2, +any_find_slice(PyObject* s1, PyObject* s2, Py_ssize_t start, - Py_ssize_t end) + Py_ssize_t end, + int direction) { int kind1, kind2; void *buf1, *buf2; @@ -9334,54 +9338,35 @@ PyUnicode_Count(PyObject *str, Py_ssize_t end) { Py_ssize_t result; - PyObject* str_obj; - PyObject* sub_obj; int kind1, kind2; void *buf1 = NULL, *buf2 = NULL; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str); - if (!str_obj) - return -1; - sub_obj = PyUnicode_FromObject(substr); - if (!sub_obj) { - Py_DECREF(str_obj); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - } - if (PyUnicode_READY(sub_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - return -1; - } - kind1 = PyUnicode_KIND(str_obj); - kind2 = PyUnicode_KIND(sub_obj); - if (kind1 < kind2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + kind1 = PyUnicode_KIND(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } - len1 = PyUnicode_GET_LENGTH(str_obj); - len2 = PyUnicode_GET_LENGTH(sub_obj); + len1 = PyUnicode_GET_LENGTH(str); + len2 = PyUnicode_GET_LENGTH(substr); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(sub_obj); - Py_DECREF(str_obj); + if (end - start < len2) return 0; - } - buf1 = PyUnicode_DATA(str_obj); - buf2 = PyUnicode_DATA(sub_obj); + buf1 = PyUnicode_DATA(str); + buf2 = PyUnicode_DATA(substr); if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub_obj, kind1); + buf2 = _PyUnicode_AsKind(substr, kind1); if (!buf2) goto onError; } switch (kind1) { case PyUnicode_1BYTE_KIND: - if (PyUnicode_IS_ASCII(str_obj) && PyUnicode_IS_ASCII(sub_obj)) + if (PyUnicode_IS_ASCII(str) && PyUnicode_IS_ASCII(substr)) result = asciilib_count( ((Py_UCS1*)buf1) + start, end - start, buf2, len2, PY_SSIZE_T_MAX @@ -9408,16 +9393,11 @@ PyUnicode_Count(PyObject *str, assert(0); result = 0; } - Py_DECREF(sub_obj); - Py_DECREF(str_obj); - if (kind2 != kind1) PyMem_Free(buf2); return result; onError: - Py_DECREF(sub_obj); - Py_DECREF(str_obj); if (kind2 != kind1 && buf2) PyMem_Free(buf2); return -1; @@ -9425,35 +9405,15 @@ PyUnicode_Count(PyObject *str, Py_ssize_t PyUnicode_Find(PyObject *str, - PyObject *sub, + PyObject *substr, Py_ssize_t start, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (!str) - return -2; - sub = PyUnicode_FromObject(sub); - if (!sub) { - Py_DECREF(str); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -2; - } - if (PyUnicode_READY(sub) == -1 || PyUnicode_READY(str) == -1) { - Py_DECREF(sub); - Py_DECREF(str); - return -2; - } - result = any_find_slice(direction, - str, sub, start, end - ); - - Py_DECREF(str); - Py_DECREF(sub); - - return result; + return any_find_slice(str, substr, start, end, direction); } Py_ssize_t @@ -9556,22 +9516,10 @@ PyUnicode_Tailmatch(PyObject *str, Py_ssize_t end, int direction) { - Py_ssize_t result; - - str = PyUnicode_FromObject(str); - if (str == NULL) + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0) return -1; - substr = PyUnicode_FromObject(substr); - if (substr == NULL) { - Py_DECREF(str); - return -1; - } - result = tailmatch(str, substr, - start, end, direction); - Py_DECREF(str); - Py_DECREF(substr); - return result; + return tailmatch(str, substr, start, end, direction); } /* Apply fixfct filter to the Unicode object self and return a @@ -10177,13 +10125,8 @@ PyUnicode_Splitlines(PyObject *string, int keepends) { PyObject *list; - string = PyUnicode_FromObject(string); - if (string == NULL) + if (ensure_unicode(string) < 0) return NULL; - if (PyUnicode_READY(string) == -1) { - Py_DECREF(string); - return NULL; - } switch (PyUnicode_KIND(string)) { case PyUnicode_1BYTE_KIND: @@ -10210,7 +10153,6 @@ PyUnicode_Splitlines(PyObject *string, int keepends) assert(0); list = 0; } - Py_DECREF(string); return list; } @@ -10771,28 +10713,27 @@ unicode_casefold(PyObject *self) } -/* Argument converter. Coerces to a single unicode character */ +/* Argument converter. Accepts a single Unicode character. */ static int convert_uc(PyObject *obj, void *addr) { Py_UCS4 *fillcharloc = (Py_UCS4 *)addr; - PyObject *uniobj; - uniobj = PyUnicode_FromObject(obj); - if (uniobj == NULL) { - PyErr_SetString(PyExc_TypeError, - "The fill character cannot be converted to Unicode"); + if (!PyUnicode_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "The fill character must be a unicode character, " + "not %.100s", Py_TYPE(obj)->tp_name); return 0; } - if (PyUnicode_GET_LENGTH(uniobj) != 1) { + if (PyUnicode_READY(obj) < 0) + return 0; + if (PyUnicode_GET_LENGTH(obj) != 1) { PyErr_SetString(PyExc_TypeError, "The fill character must be exactly one character long"); - Py_DECREF(uniobj); return 0; } - *fillcharloc = PyUnicode_READ_CHAR(uniobj, 0); - Py_DECREF(uniobj); + *fillcharloc = PyUnicode_READ_CHAR(obj, 0); return 1; } @@ -11114,59 +11055,43 @@ _PyUnicode_EQ(PyObject *aa, PyObject *bb) } int -PyUnicode_Contains(PyObject *container, PyObject *element) +PyUnicode_Contains(PyObject *str, PyObject *substr) { - PyObject *str, *sub; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; int result; - /* Coerce the two arguments */ - sub = PyUnicode_FromObject(element); - if (!sub) { + if (!PyUnicode_Check(substr)) { PyErr_Format(PyExc_TypeError, - "'in ' requires string as left operand, not %s", - element->ob_type->tp_name); + "'in ' requires string as left operand, not %.100s", + Py_TYPE(substr)->tp_name); return -1; } - - str = PyUnicode_FromObject(container); - if (!str) { - Py_DECREF(sub); + if (PyUnicode_READY(substr) == -1) + return -1; + if (ensure_unicode(str) < 0) return -1; - } kind1 = PyUnicode_KIND(str); - kind2 = PyUnicode_KIND(sub); - if (kind1 < kind2) { - Py_DECREF(sub); - Py_DECREF(str); + kind2 = PyUnicode_KIND(substr); + if (kind1 < kind2) return 0; - } len1 = PyUnicode_GET_LENGTH(str); - len2 = PyUnicode_GET_LENGTH(sub); - if (len1 < len2) { - Py_DECREF(sub); - Py_DECREF(str); + len2 = PyUnicode_GET_LENGTH(substr); + if (len1 < len2) return 0; - } buf1 = PyUnicode_DATA(str); - buf2 = PyUnicode_DATA(sub); + buf2 = PyUnicode_DATA(substr); if (len2 == 1) { Py_UCS4 ch = PyUnicode_READ(kind2, buf2, 0); result = findchar((const char *)buf1, kind1, len1, ch, 1) != -1; - Py_DECREF(sub); - Py_DECREF(str); return result; } if (kind2 != kind1) { - buf2 = _PyUnicode_AsKind(sub, kind1); - if (!buf2) { - Py_DECREF(sub); - Py_DECREF(str); + buf2 = _PyUnicode_AsKind(substr, kind1); + if (!buf2) return -1; - } } switch (kind1) { @@ -11184,9 +11109,6 @@ PyUnicode_Contains(PyObject *container, PyObject *element) assert(0); } - Py_DECREF(str); - Py_DECREF(sub); - if (kind2 != kind1) PyMem_Free(buf2); @@ -11198,56 +11120,40 @@ PyUnicode_Contains(PyObject *container, PyObject *element) PyObject * PyUnicode_Concat(PyObject *left, PyObject *right) { - PyObject *u = NULL, *v = NULL, *w; + PyObject *result; Py_UCS4 maxchar, maxchar2; - Py_ssize_t u_len, v_len, new_len; + Py_ssize_t left_len, right_len, new_len; - /* Coerce the two arguments */ - u = PyUnicode_FromObject(left); - if (u == NULL) - goto onError; - v = PyUnicode_FromObject(right); - if (v == NULL) - goto onError; + if (ensure_unicode(left) < 0 || ensure_unicode(right) < 0) + return NULL; /* Shortcuts */ - if (v == unicode_empty) { - Py_DECREF(v); - return u; - } - if (u == unicode_empty) { - Py_DECREF(u); - return v; - } + if (left == unicode_empty) + return PyUnicode_FromObject(right); + if (right == unicode_empty) + return PyUnicode_FromObject(left); - u_len = PyUnicode_GET_LENGTH(u); - v_len = PyUnicode_GET_LENGTH(v); - if (u_len > PY_SSIZE_T_MAX - v_len) { + left_len = PyUnicode_GET_LENGTH(left); + right_len = PyUnicode_GET_LENGTH(right); + if (left_len > PY_SSIZE_T_MAX - right_len) { PyErr_SetString(PyExc_OverflowError, "strings are too large to concat"); - goto onError; + return NULL; } - new_len = u_len + v_len; + new_len = left_len + right_len; - maxchar = PyUnicode_MAX_CHAR_VALUE(u); - maxchar2 = PyUnicode_MAX_CHAR_VALUE(v); + maxchar = PyUnicode_MAX_CHAR_VALUE(left); + maxchar2 = PyUnicode_MAX_CHAR_VALUE(right); maxchar = Py_MAX(maxchar, maxchar2); /* Concat the two Unicode strings */ - w = PyUnicode_New(new_len, maxchar); - if (w == NULL) - goto onError; - _PyUnicode_FastCopyCharacters(w, 0, u, 0, u_len); - _PyUnicode_FastCopyCharacters(w, u_len, v, 0, v_len); - Py_DECREF(u); - Py_DECREF(v); - assert(_PyUnicode_CheckConsistency(w, 1)); - return w; - - onError: - Py_XDECREF(u); - Py_XDECREF(v); - return NULL; + result = PyUnicode_New(new_len, maxchar); + if (result == NULL) + return NULL; + _PyUnicode_FastCopyCharacters(result, 0, left, 0, left_len); + _PyUnicode_FastCopyCharacters(result, left_len, right, 0, right_len); + assert(_PyUnicode_CheckConsistency(result, 1)); + return result; } void @@ -11362,25 +11268,21 @@ unicode_count(PyObject *self, PyObject *args) kind1 = PyUnicode_KIND(self); kind2 = PyUnicode_KIND(substring); - if (kind1 < kind2) { - Py_DECREF(substring); + if (kind1 < kind2) return PyLong_FromLong(0); - } + len1 = PyUnicode_GET_LENGTH(self); len2 = PyUnicode_GET_LENGTH(substring); ADJUST_INDICES(start, end, len1); - if (end - start < len2) { - Py_DECREF(substring); + if (end - start < len2) return PyLong_FromLong(0); - } + buf1 = PyUnicode_DATA(self); buf2 = PyUnicode_DATA(substring); if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(substring, kind1); - if (!buf2) { - Py_DECREF(substring); + if (!buf2) return NULL; - } } switch (kind1) { case PyUnicode_1BYTE_KIND: @@ -11410,8 +11312,6 @@ unicode_count(PyObject *self, PyObject *args) if (kind2 != kind1) PyMem_Free(buf2); - Py_DECREF(substring); - return result; } @@ -11549,18 +11449,10 @@ unicode_find(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -11637,18 +11529,10 @@ unicode_index(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - result = any_find_slice(1, self, substring, start, end); - - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, 1); if (result == -2) return NULL; @@ -12457,40 +12341,15 @@ unicode_repeat(PyObject *str, Py_ssize_t len) } PyObject * -PyUnicode_Replace(PyObject *obj, - PyObject *subobj, - PyObject *replobj, +PyUnicode_Replace(PyObject *str, + PyObject *substr, + PyObject *replstr, Py_ssize_t maxcount) { - PyObject *self; - PyObject *str1; - PyObject *str2; - PyObject *result; - - self = PyUnicode_FromObject(obj); - if (self == NULL) - return NULL; - str1 = PyUnicode_FromObject(subobj); - if (str1 == NULL) { - Py_DECREF(self); - return NULL; - } - str2 = PyUnicode_FromObject(replobj); - if (str2 == NULL) { - Py_DECREF(self); - Py_DECREF(str1); + if (ensure_unicode(str) < 0 || ensure_unicode(substr) < 0 || + ensure_unicode(replstr) < 0) return NULL; - } - if (PyUnicode_READY(self) == -1 || - PyUnicode_READY(str1) == -1 || - PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - Py_DECREF(self); - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(str, substr, replstr, maxcount); } PyDoc_STRVAR(replace__doc__, @@ -12506,28 +12365,12 @@ unicode_replace(PyObject *self, PyObject *args) PyObject *str1; PyObject *str2; Py_ssize_t maxcount = -1; - PyObject *result; - if (!PyArg_ParseTuple(args, "OO|n:replace", &str1, &str2, &maxcount)) + if (!PyArg_ParseTuple(args, "UU|n:replace", &str1, &str2, &maxcount)) return NULL; if (PyUnicode_READY(self) == -1) return NULL; - str1 = PyUnicode_FromObject(str1); - if (str1 == NULL) - return NULL; - str2 = PyUnicode_FromObject(str2); - if (str2 == NULL) { - Py_DECREF(str1); - return NULL; - } - if (PyUnicode_READY(str1) == -1 || PyUnicode_READY(str2) == -1) - result = NULL; - else - result = replace(self, str1, str2, maxcount); - - Py_DECREF(str1); - Py_DECREF(str2); - return result; + return replace(self, str1, str2, maxcount); } static PyObject * @@ -12716,18 +12559,10 @@ unicode_rfind(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(-1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12753,18 +12588,10 @@ unicode_rindex(PyObject *self, PyObject *args) &start, &end)) return NULL; - if (PyUnicode_READY(self) == -1) { - Py_DECREF(substring); - return NULL; - } - if (PyUnicode_READY(substring) == -1) { - Py_DECREF(substring); + if (PyUnicode_READY(self) == -1) return NULL; - } - - result = any_find_slice(-1, self, substring, start, end); - Py_DECREF(substring); + result = any_find_slice(self, substring, start, end, -1); if (result == -2) return NULL; @@ -12804,24 +12631,10 @@ unicode_rjust(PyObject *self, PyObject *args) PyObject * PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - result = split(s, sep, maxsplit); - - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return split(s, sep, maxsplit); } PyDoc_STRVAR(split__doc__, @@ -12846,35 +12659,26 @@ unicode_split(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return split(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return split(self, substring, maxcount); - else - return PyUnicode_Split(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyObject * -PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) +PyUnicode_Partition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) - return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - } - if (PyUnicode_READY(sep_obj) == -1 || PyUnicode_READY(str_obj) == -1) { - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12888,8 +12692,6 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, str_obj, unicode_empty, unicode_empty); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12897,7 +12699,7 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12918,39 +12720,23 @@ PyUnicode_Partition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyObject * -PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) +PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) { - PyObject* str_obj; - PyObject* sep_obj; PyObject* out; int kind1, kind2; void *buf1, *buf2; Py_ssize_t len1, len2; - str_obj = PyUnicode_FromObject(str_in); - if (!str_obj) + if (ensure_unicode(str_obj) < 0 || ensure_unicode(sep_obj) < 0) return NULL; - sep_obj = PyUnicode_FromObject(sep_in); - if (!sep_obj) { - Py_DECREF(str_obj); - return NULL; - } kind1 = PyUnicode_KIND(str_obj); kind2 = PyUnicode_KIND(sep_obj); @@ -12964,8 +12750,6 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = PyTuple_Pack(3, unicode_empty, unicode_empty, str_obj); Py_DECREF(unicode_empty); } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); return out; } buf1 = PyUnicode_DATA(str_obj); @@ -12973,7 +12757,7 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) if (kind2 != kind1) { buf2 = _PyUnicode_AsKind(sep_obj, kind1); if (!buf2) - goto onError; + return NULL; } switch (kind1) { @@ -12994,18 +12778,10 @@ PyUnicode_RPartition(PyObject *str_in, PyObject *sep_in) out = 0; } - Py_DECREF(sep_obj); - Py_DECREF(str_obj); if (kind2 != kind1) PyMem_Free(buf2); return out; - onError: - Py_DECREF(sep_obj); - Py_DECREF(str_obj); - if (kind2 != kind1 && buf2) - PyMem_Free(buf2); - return NULL; } PyDoc_STRVAR(partition__doc__, @@ -13037,24 +12813,10 @@ unicode_rpartition(PyObject *self, PyObject *separator) PyObject * PyUnicode_RSplit(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) { - PyObject *result; - - s = PyUnicode_FromObject(s); - if (s == NULL) + if (ensure_unicode(s) < 0 || (sep != NULL && ensure_unicode(sep) < 0)) return NULL; - if (sep != NULL) { - sep = PyUnicode_FromObject(sep); - if (sep == NULL) { - Py_DECREF(s); - return NULL; - } - } - result = rsplit(s, sep, maxsplit); - - Py_DECREF(s); - Py_XDECREF(sep); - return result; + return rsplit(s, sep, maxsplit); } PyDoc_STRVAR(rsplit__doc__, @@ -13079,10 +12841,14 @@ unicode_rsplit(PyObject *self, PyObject *args, PyObject *kwds) if (substring == Py_None) return rsplit(self, NULL, maxcount); - else if (PyUnicode_Check(substring)) + + if (PyUnicode_Check(substring)) return rsplit(self, substring, maxcount); - else - return PyUnicode_RSplit(self, substring, maxcount); + + PyErr_Format(PyExc_TypeError, + "must be str or None, not %.100s", + Py_TYPE(substring)->tp_name); + return NULL; } PyDoc_STRVAR(splitlines__doc__, @@ -13363,11 +13129,15 @@ unicode_startswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject(PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for startswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13377,15 +13147,13 @@ unicode_startswith(PyObject *self, /* nothing matched */ Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "startswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "startswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, -1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, -1); if (result == -1) return NULL; return PyBool_FromLong(result); @@ -13415,12 +13183,15 @@ unicode_endswith(PyObject *self, if (PyTuple_Check(subobj)) { Py_ssize_t i; for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - substring = PyUnicode_FromObject( - PyTuple_GET_ITEM(subobj, i)); - if (substring == NULL) + substring = PyTuple_GET_ITEM(subobj, i); + if (!PyUnicode_Check(substring)) { + PyErr_Format(PyExc_TypeError, + "tuple for endswith must only contain str, " + "not %.100s", + Py_TYPE(substring)->tp_name); return NULL; + } result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); if (result == -1) return NULL; if (result) { @@ -13429,15 +13200,13 @@ unicode_endswith(PyObject *self, } Py_RETURN_FALSE; } - substring = PyUnicode_FromObject(subobj); - if (substring == NULL) { - if (PyErr_ExceptionMatches(PyExc_TypeError)) - PyErr_Format(PyExc_TypeError, "endswith first arg must be str or " - "a tuple of str, not %s", Py_TYPE(subobj)->tp_name); + if (!PyUnicode_Check(subobj)) { + PyErr_Format(PyExc_TypeError, + "endswith first arg must be str or " + "a tuple of str, not %.100s", Py_TYPE(subobj)->tp_name); return NULL; } - result = tailmatch(self, substring, start, end, +1); - Py_DECREF(substring); + result = tailmatch(self, subobj, start, end, +1); if (result == -1) return NULL; return PyBool_FromLong(result); @@ -14907,13 +14676,10 @@ PyUnicode_Format(PyObject *format, PyObject *args) return NULL; } - ctx.fmtstr = PyUnicode_FromObject(format); - if (ctx.fmtstr == NULL) + if (ensure_unicode(format) < 0) return NULL; - if (PyUnicode_READY(ctx.fmtstr) == -1) { - Py_DECREF(ctx.fmtstr); - return NULL; - } + + ctx.fmtstr = format; ctx.fmtdata = PyUnicode_DATA(ctx.fmtstr); ctx.fmtkind = PyUnicode_KIND(ctx.fmtstr); ctx.fmtcnt = PyUnicode_GET_LENGTH(ctx.fmtstr); @@ -14973,11 +14739,9 @@ PyUnicode_Format(PyObject *format, PyObject *args) if (ctx.args_owned) { Py_DECREF(ctx.args); } - Py_DECREF(ctx.fmtstr); return _PyUnicodeWriter_Finish(&ctx.writer); onError: - Py_DECREF(ctx.fmtstr); _PyUnicodeWriter_Dealloc(&ctx.writer); if (ctx.args_owned) { Py_DECREF(ctx.args); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 31d9e0e..29fcffe 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1931,9 +1931,8 @@ builtin_input_impl(PyModuleDef *module, PyObject *prompt) Py_CLEAR(stringpo); if (po == NULL) goto _readline_errors; - promptstr = PyBytes_AsString(po); - if (promptstr == NULL) - goto _readline_errors; + assert(PyBytes_Check(po)); + promptstr = PyBytes_AS_STRING(po); } else { po = NULL; diff --git a/Python/getargs.c b/Python/getargs.c index 05ec27b..9858bd5 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1056,35 +1056,25 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return converterr("(AsCharBuffer failed)", arg, msgbuf, bufsize); } - else { - PyObject *u; - - /* Convert object to Unicode */ - u = PyUnicode_FromObject(arg); - if (u == NULL) - return converterr( - "string or unicode or text buffer", - arg, msgbuf, bufsize); - + else if (PyUnicode_Check(arg)) { /* Encode object; use default error handling */ - s = PyUnicode_AsEncodedString(u, + s = PyUnicode_AsEncodedString(arg, encoding, NULL); - Py_DECREF(u); if (s == NULL) return converterr("(encoding failed)", arg, msgbuf, bufsize); - if (!PyBytes_Check(s)) { - Py_DECREF(s); - return converterr( - "(encoder failed to return bytes)", - arg, msgbuf, bufsize); - } + assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); ptr = PyBytes_AS_STRING(s); if (ptr == NULL) ptr = ""; } + else { + return converterr( + recode_strings ? "str" : "str, bytes or bytearray", + arg, msgbuf, bufsize); + } /* Write output; output is guaranteed to be 0-terminated */ if (*format == '#') { -- cgit v0.12 From 23c5cbbddec66063f1cc44cc06045a39a119b3d6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 14 Apr 2016 12:36:11 +0300 Subject: fs_unicode_converter is no longer used. --- Python/import.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Python/import.c b/Python/import.c index 0608924..6d71f2a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -38,14 +38,6 @@ module _imp #include "clinic/import.c.h" -/*[python input] -class fs_unicode_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSDecoder' - -[python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=9d6786230166006e]*/ - /* Initialize things */ void -- cgit v0.12 From 95702725ff7ce93c332757fec6ba00a59f5ded94 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 01:51:31 +1000 Subject: Add secrets module and tests. --- Lib/secrets.py | 149 +++++++++++++++++++++++++++++++++++++++++++++++ Lib/test/test_secrets.py | 120 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) create mode 100644 Lib/secrets.py create mode 100644 Lib/test/test_secrets.py diff --git a/Lib/secrets.py b/Lib/secrets.py new file mode 100644 index 0000000..ed018ad --- /dev/null +++ b/Lib/secrets.py @@ -0,0 +1,149 @@ +"""Generate cryptographically strong pseudo-random numbers suitable for +managing secrets such as account authentication, tokens, and similar. +See PEP 506 for more information. + +https://www.python.org/dev/peps/pep-0506/ + + +Random numbers +============== + +The ``secrets`` module provides the following pseudo-random functions, based +on SystemRandom, which in turn uses the most secure source of randomness your +operating system provides. + + + choice(sequence) + Choose a random element from a non-empty sequence. + + randbelow(n) + Return a random int in the range [0, n). + + randbits(k) + Generates an int with k random bits. + + SystemRandom + Class for generating random numbers using sources provided by + the operating system. See the ``random`` module for documentation. + + +Token functions +=============== + +The ``secrets`` module provides a number of functions for generating secure +tokens, suitable for applications such as password resets, hard-to-guess +URLs, and similar. All the ``token_*`` functions take an optional single +argument specifying the number of bytes of randomness to use. If that is +not given, or is ``None``, a reasonable default is used. That default is +subject to change at any time, including during maintenance releases. + + + token_bytes(nbytes=None) + Return a random byte-string containing ``nbytes`` number of bytes. + + >>> secrets.token_bytes(16) #doctest:+SKIP + b'\\xebr\\x17D*t\\xae\\xd4\\xe3S\\xb6\\xe2\\xebP1\\x8b' + + + token_hex(nbytes=None) + Return a random text-string, in hexadecimal. The string has ``nbytes`` + random bytes, each byte converted to two hex digits. + + >>> secrets.token_hex(16) #doctest:+SKIP + 'f9bf78b9a18ce6d46a0cd2b0b86df9da' + + token_urlsafe(nbytes=None) + Return a random URL-safe text-string, containing ``nbytes`` random + bytes. On average, each byte results in approximately 1.3 characters + in the final result. + + >>> secrets.token_urlsafe(16) #doctest:+SKIP + 'Drmhze6EPcv0fN_81Bj-nA' + + +(The examples above assume Python 3. In Python 2, byte-strings will display +using regular quotes ``''`` with no prefix, and text-strings will have a +``u`` prefix.) + + +Other functions +=============== + + compare_digest(a, b) + Return True if strings a and b are equal, otherwise False. + Performs the equality comparison in such a way as to reduce the + risk of timing attacks. + + See http://codahale.com/a-lesson-in-timing-attacks/ for a + discussion on how timing attacks against ``==`` can reveal + secrets from your application. + + +""" + +__all__ = ['choice', 'randbelow', 'randbits', 'SystemRandom', + 'token_bytes', 'token_hex', 'token_urlsafe', + 'compare_digest', + ] + + +import base64 +import binascii +import os + +try: + from hmac import compare_digest +except ImportError: + # Python version is too old. Fall back to a pure-Python version. + + import operator + from functools import reduce + + def compare_digest(a, b): + """Return ``a == b`` using an approach resistant to timing analysis. + + a and b must both be of the same type: either both text strings, + or both byte strings. + + Note: If a and b are of different lengths, or if an error occurs, + a timing attack could theoretically reveal information about the + types and lengths of a and b, but not their values. + """ + # For a similar approach, see + # http://codahale.com/a-lesson-in-timing-attacks/ + for T in (bytes, str): + if isinstance(a, T) and isinstance(b, T): + break + else: # for...else + raise TypeError("arguments must be both strings or both bytes") + if len(a) != len(b): + return False + # Thanks to Raymond Hettinger for this one-liner. + return reduce(operator.and_, map(operator.eq, a, b), True) + + + +from random import SystemRandom + +_sysrand = SystemRandom() + +randbits = _sysrand.getrandbits +choice = _sysrand.choice + +def randbelow(exclusive_upper_bound): + return _sysrand._randbelow(exclusive_upper_bound) + +DEFAULT_ENTROPY = 32 # number of bytes to return by default + +def token_bytes(nbytes=None): + if nbytes is None: + nbytes = DEFAULT_ENTROPY + return os.urandom(nbytes) + +def token_hex(nbytes=None): + return binascii.hexlify(token_bytes(nbytes)).decode('ascii') + +def token_urlsafe(nbytes=None): + tok = token_bytes(nbytes) + return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii') + diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py new file mode 100644 index 0000000..a3d1a8c --- /dev/null +++ b/Lib/test/test_secrets.py @@ -0,0 +1,120 @@ +"""Test the secrets module. + +As most of the functions in secrets are thin wrappers around functions +defined elsewhere, we don't need to test them exhaustively. +""" + + +import secrets +import unittest +import string + + +# === Unit tests === + +class Compare_Digest_Tests(unittest.TestCase): + """Test secrets.compare_digest function.""" + + def test_equal(self): + # Test compare_digest functionality with equal (byte/text) strings. + for s in ("a", "bcd", "xyz123"): + a = s*100 + b = s*100 + self.assertTrue(secrets.compare_digest(a, b)) + self.assertTrue(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_unequal(self): + # Test compare_digest functionality with unequal (byte/text) strings. + self.assertFalse(secrets.compare_digest("abc", "abcd")) + self.assertFalse(secrets.compare_digest(b"abc", b"abcd")) + for s in ("x", "mn", "a1b2c3"): + a = s*100 + "q" + b = s*100 + "k" + self.assertFalse(secrets.compare_digest(a, b)) + self.assertFalse(secrets.compare_digest(a.encode('utf-8'), b.encode('utf-8'))) + + def test_bad_types(self): + # Test that compare_digest raises with mixed types. + a = 'abcde' + b = a.encode('utf-8') + assert isinstance(a, str) + assert isinstance(b, bytes) + self.assertRaises(TypeError, secrets.compare_digest, a, b) + self.assertRaises(TypeError, secrets.compare_digest, b, a) + + def test_bool(self): + # Test that compare_digest returns a bool. + self.assertTrue(isinstance(secrets.compare_digest("abc", "abc"), bool)) + self.assertTrue(isinstance(secrets.compare_digest("abc", "xyz"), bool)) + + +class Random_Tests(unittest.TestCase): + """Test wrappers around SystemRandom methods.""" + + def test_randbits(self): + # Test randbits. + errmsg = "randbits(%d) returned %d" + for numbits in (3, 12, 30): + for i in range(6): + n = secrets.randbits(numbits) + self.assertTrue(0 <= n < 2**numbits, errmsg % (numbits, n)) + + def test_choice(self): + # Test choice. + items = [1, 2, 4, 8, 16, 32, 64] + for i in range(10): + self.assertTrue(secrets.choice(items) in items) + + def test_randbelow(self): + # Test randbelow. + errmsg = "randbelow(%d) returned %d" + for i in range(2, 10): + n = secrets.randbelow(i) + self.assertTrue(n in range(i), errmsg % (i, n)) + self.assertRaises(ValueError, secrets.randbelow, 0) + + +class Token_Tests(unittest.TestCase): + """Test token functions.""" + + def test_token_defaults(self): + # Test that token_* functions handle default size correctly. + for func in (secrets.token_bytes, secrets.token_hex, + secrets.token_urlsafe): + name = func.__name__ + try: + func() + except TypeError: + self.fail("%s cannot be called with no argument" % name) + try: + func(None) + except TypeError: + self.fail("%s cannot be called with None" % name) + size = secrets.DEFAULT_ENTROPY + self.assertEqual(len(secrets.token_bytes(None)), size) + self.assertEqual(len(secrets.token_hex(None)), 2*size) + + def test_token_bytes(self): + # Test token_bytes. + self.assertTrue(isinstance(secrets.token_bytes(11), bytes)) + for n in (1, 8, 17, 100): + self.assertEqual(len(secrets.token_bytes(n)), n) + + def test_token_hex(self): + # Test token_hex. + self.assertTrue(isinstance(secrets.token_hex(7), str)) + for n in (1, 12, 25, 90): + s = secrets.token_hex(n) + self.assertEqual(len(s), 2*n) + self.assertTrue(all(c in string.hexdigits for c in s)) + + def test_token_urlsafe(self): + # Test token_urlsafe. + self.assertTrue(isinstance(secrets.token_urlsafe(9), str)) + legal = string.ascii_letters + string.digits + '-_' + for n in (1, 11, 28, 76): + self.assertTrue(all(c in legal for c in secrets.token_urlsafe(n))) + + +if __name__ == '__main__': + unittest.main() -- cgit v0.12 From a873f6824886007ca1096b9d3a0a730c52c1c327 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 01:55:14 +1000 Subject: run Tools/reindent.py on secrets.py to satisfy the checkwhitespace hook --- Lib/secrets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/secrets.py b/Lib/secrets.py index ed018ad..e0f2656 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -146,4 +146,3 @@ def token_hex(nbytes=None): def token_urlsafe(nbytes=None): tok = token_bytes(nbytes) return base64.urlsafe_b64encode(tok).rstrip(b'=').decode('ascii') - -- cgit v0.12 From 08fbef040a02d684c1c8466b98f8d68d38c3de47 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 10:04:24 +1000 Subject: Improve tests with more modern assert* methods and subTests. --- Lib/test/test_secrets.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py index a3d1a8c..afcba84 100644 --- a/Lib/test/test_secrets.py +++ b/Lib/test/test_secrets.py @@ -44,8 +44,8 @@ class Compare_Digest_Tests(unittest.TestCase): def test_bool(self): # Test that compare_digest returns a bool. - self.assertTrue(isinstance(secrets.compare_digest("abc", "abc"), bool)) - self.assertTrue(isinstance(secrets.compare_digest("abc", "xyz"), bool)) + self.assertIsInstance(secrets.compare_digest("abc", "abc"), bool) + self.assertIsInstance(secrets.compare_digest("abc", "xyz"), bool) class Random_Tests(unittest.TestCase): @@ -67,10 +67,8 @@ class Random_Tests(unittest.TestCase): def test_randbelow(self): # Test randbelow. - errmsg = "randbelow(%d) returned %d" for i in range(2, 10): - n = secrets.randbelow(i) - self.assertTrue(n in range(i), errmsg % (i, n)) + self.assertIn(secrets.randbelow(i), range(i)) self.assertRaises(ValueError, secrets.randbelow, 0) @@ -81,39 +79,44 @@ class Token_Tests(unittest.TestCase): # Test that token_* functions handle default size correctly. for func in (secrets.token_bytes, secrets.token_hex, secrets.token_urlsafe): - name = func.__name__ - try: - func() - except TypeError: - self.fail("%s cannot be called with no argument" % name) - try: - func(None) - except TypeError: - self.fail("%s cannot be called with None" % name) + with self.subTest(func=func): + name = func.__name__ + try: + func() + except TypeError: + self.fail("%s cannot be called with no argument" % name) + try: + func(None) + except TypeError: + self.fail("%s cannot be called with None" % name) size = secrets.DEFAULT_ENTROPY self.assertEqual(len(secrets.token_bytes(None)), size) self.assertEqual(len(secrets.token_hex(None)), 2*size) def test_token_bytes(self): # Test token_bytes. - self.assertTrue(isinstance(secrets.token_bytes(11), bytes)) for n in (1, 8, 17, 100): - self.assertEqual(len(secrets.token_bytes(n)), n) + with self.subTest(n=n): + self.assertIsInstance(secrets.token_bytes(n), bytes) + self.assertEqual(len(secrets.token_bytes(n)), n) def test_token_hex(self): # Test token_hex. - self.assertTrue(isinstance(secrets.token_hex(7), str)) for n in (1, 12, 25, 90): - s = secrets.token_hex(n) - self.assertEqual(len(s), 2*n) - self.assertTrue(all(c in string.hexdigits for c in s)) + with self.subTest(n=n): + s = secrets.token_hex(n) + self.assertIsInstance(s, str)) + self.assertEqual(len(s), 2*n) + self.assertTrue(all(c in string.hexdigits for c in s)) def test_token_urlsafe(self): # Test token_urlsafe. - self.assertTrue(isinstance(secrets.token_urlsafe(9), str)) legal = string.ascii_letters + string.digits + '-_' for n in (1, 11, 28, 76): - self.assertTrue(all(c in legal for c in secrets.token_urlsafe(n))) + with self.subTest(n=n): + s = secrets.token_urlsafe(n) + self.assertIsInstance(s, str)) + self.assertTrue(all(c in legal for c in s)) if __name__ == '__main__': -- cgit v0.12 From 8ca020e1cc2590472ba83611727ae20d9f34bfe5 Mon Sep 17 00:00:00 2001 From: Steven D'Aprano Date: Fri, 15 Apr 2016 10:06:18 +1000 Subject: Fix missing parens. --- Lib/test/test_secrets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_secrets.py b/Lib/test/test_secrets.py index afcba84..4c65cf0 100644 --- a/Lib/test/test_secrets.py +++ b/Lib/test/test_secrets.py @@ -105,7 +105,7 @@ class Token_Tests(unittest.TestCase): for n in (1, 12, 25, 90): with self.subTest(n=n): s = secrets.token_hex(n) - self.assertIsInstance(s, str)) + self.assertIsInstance(s, str) self.assertEqual(len(s), 2*n) self.assertTrue(all(c in string.hexdigits for c in s)) @@ -115,7 +115,7 @@ class Token_Tests(unittest.TestCase): for n in (1, 11, 28, 76): with self.subTest(n=n): s = secrets.token_urlsafe(n) - self.assertIsInstance(s, str)) + self.assertIsInstance(s, str) self.assertTrue(all(c in legal for c in s)) -- cgit v0.12 From c9a59e6e4f221f492578a03546e3d7c96b9da1b8 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 15 Apr 2016 14:11:10 +0300 Subject: Issue #26764: Fixed SystemError in bytes.__rmod__. --- Lib/test/test_bytes.py | 37 +++++++++++-------------------------- Objects/bytesobject.c | 10 ++++------ 2 files changed, 15 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index bd3410f..966e287 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -483,26 +483,33 @@ class BaseBytesTest: self.assertRaises(ValueError, b.rindex, w, 1, 3) def test_mod(self): - b = b'hello, %b!' + b = self.type2test(b'hello, %b!') orig = b b = b % b'world' self.assertEqual(b, b'hello, world!') self.assertEqual(orig, b'hello, %b!') self.assertFalse(b is orig) - b = b'%s / 100 = %d%%' + b = self.type2test(b'%s / 100 = %d%%') a = b % (b'seventy-nine', 79) self.assertEqual(a, b'seventy-nine / 100 = 79%') + self.assertIs(type(a), bytes) def test_imod(self): - b = b'hello, %b!' + b = self.type2test(b'hello, %b!') orig = b b %= b'world' self.assertEqual(b, b'hello, world!') self.assertEqual(orig, b'hello, %b!') self.assertFalse(b is orig) - b = b'%s / 100 = %d%%' + b = self.type2test(b'%s / 100 = %d%%') b %= (b'seventy-nine', 79) self.assertEqual(b, b'seventy-nine / 100 = 79%') + self.assertIs(type(b), bytes) + + def test_rmod(self): + with self.assertRaises(TypeError): + object() % self.type2test(b'abc') + self.assertIs(self.type2test(b'abc').__rmod__('%r'), NotImplemented) def test_replace(self): b = self.type2test(b'mississippi') @@ -1064,28 +1071,6 @@ class ByteArrayTest(BaseBytesTest, unittest.TestCase): b[8:] = b self.assertEqual(b, bytearray(list(range(8)) + list(range(256)))) - def test_mod(self): - b = bytearray(b'hello, %b!') - orig = b - b = b % b'world' - self.assertEqual(b, b'hello, world!') - self.assertEqual(orig, bytearray(b'hello, %b!')) - self.assertFalse(b is orig) - b = bytearray(b'%s / 100 = %d%%') - a = b % (b'seventy-nine', 79) - self.assertEqual(a, bytearray(b'seventy-nine / 100 = 79%')) - - def test_imod(self): - b = bytearray(b'hello, %b!') - orig = b - b %= b'world' - self.assertEqual(b, b'hello, world!') - self.assertEqual(orig, bytearray(b'hello, %b!')) - self.assertFalse(b is orig) -