diff options
Diffstat (limited to 'Include')
44 files changed, 1229 insertions, 541 deletions
diff --git a/Include/Python-ast.h b/Include/Python-ast.h index 0ad788b..7ad6cb3 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -36,6 +36,8 @@ typedef struct _keyword *keyword_ty; typedef struct _alias *alias_ty; +typedef struct _withitem *withitem_ty; + enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, Suite_kind=4}; @@ -64,10 +66,9 @@ struct _mod { enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3, Delete_kind=4, Assign_kind=5, AugAssign_kind=6, For_kind=7, While_kind=8, If_kind=9, With_kind=10, Raise_kind=11, - TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14, - Import_kind=15, ImportFrom_kind=16, Global_kind=17, - Nonlocal_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21, - Continue_kind=22}; + Try_kind=12, Assert_kind=13, Import_kind=14, + ImportFrom_kind=15, Global_kind=16, Nonlocal_kind=17, + Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21}; struct _stmt { enum _stmt_kind kind; union { @@ -128,8 +129,7 @@ struct _stmt { } If; struct { - expr_ty context_expr; - expr_ty optional_vars; + asdl_seq *items; asdl_seq *body; } With; @@ -142,12 +142,8 @@ struct _stmt { asdl_seq *body; asdl_seq *handlers; asdl_seq *orelse; - } TryExcept; - - struct { - asdl_seq *body; asdl_seq *finalbody; - } TryFinally; + } Try; struct { expr_ty test; @@ -184,10 +180,10 @@ struct _stmt { enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, - Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15, - Str_kind=16, Bytes_kind=17, Ellipsis_kind=18, - Attribute_kind=19, Subscript_kind=20, Starred_kind=21, - Name_kind=22, List_kind=23, Tuple_kind=24}; + Yield_kind=12, YieldFrom_kind=13, Compare_kind=14, + Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18, + Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21, + Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25}; struct _expr { enum _expr_kind kind; union { @@ -253,6 +249,10 @@ struct _expr { } Yield; struct { + expr_ty value; + } YieldFrom; + + struct { expr_ty left; asdl_int_seq *ops; asdl_seq *comparators; @@ -275,7 +275,7 @@ struct _expr { } Str; struct { - string s; + bytes s; } Bytes; struct { @@ -383,6 +383,11 @@ struct _alias { identifier asname; }; +struct _withitem { + expr_ty context_expr; + expr_ty optional_vars; +}; + #define Module(a0, a1) _Py_Module(a0, a1) mod_ty _Py_Module(asdl_seq * body, PyArena *arena); @@ -421,18 +426,16 @@ stmt_ty _Py_While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, #define If(a0, a1, a2, a3, a4, a5) _Py_If(a0, a1, a2, a3, a4, a5) stmt_ty _Py_If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno, int col_offset, PyArena *arena); -#define With(a0, a1, a2, a3, a4, a5) _Py_With(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_With(expr_ty context_expr, expr_ty optional_vars, asdl_seq * body, - int lineno, int col_offset, PyArena *arena); +#define With(a0, a1, a2, a3, a4) _Py_With(a0, a1, a2, a3, a4) +stmt_ty _Py_With(asdl_seq * items, asdl_seq * body, int lineno, int col_offset, + PyArena *arena); #define Raise(a0, a1, a2, a3, a4) _Py_Raise(a0, a1, a2, a3, a4) stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, PyArena *arena); -#define TryExcept(a0, a1, a2, a3, a4, a5) _Py_TryExcept(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, - int lineno, int col_offset, PyArena *arena); -#define TryFinally(a0, a1, a2, a3, a4) _Py_TryFinally(a0, a1, a2, a3, a4) -stmt_ty _Py_TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, int - col_offset, PyArena *arena); +#define Try(a0, a1, a2, a3, a4, a5, a6) _Py_Try(a0, a1, a2, a3, a4, a5, a6) +stmt_ty _Py_Try(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, + asdl_seq * finalbody, int lineno, int col_offset, PyArena + *arena); #define Assert(a0, a1, a2, a3, a4) _Py_Assert(a0, a1, a2, a3, a4) stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, PyArena *arena); @@ -490,6 +493,9 @@ expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int col_offset, PyArena *arena); #define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3) expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena); +#define YieldFrom(a0, a1, a2, a3) _Py_YieldFrom(a0, a1, a2, a3) +expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena + *arena); #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5) expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno, int col_offset, PyArena *arena); @@ -502,7 +508,7 @@ expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena); #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3) expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena); #define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3) -expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena); +expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena); #define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) @@ -547,6 +553,9 @@ arg_ty _Py_arg(identifier arg, expr_ty annotation, PyArena *arena); keyword_ty _Py_keyword(identifier arg, expr_ty value, PyArena *arena); #define alias(a0, a1, a2) _Py_alias(a0, a1, a2) alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); +#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) +withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena + *arena); PyObject* PyAST_mod2obj(mod_ty t); mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); diff --git a/Include/Python.h b/Include/Python.h index 5972ffa..01b98f9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -48,6 +48,7 @@ #include <assert.h> #include "pyport.h" +#include "pymacro.h" #include "pyatomic.h" @@ -126,43 +127,6 @@ #include "pystrcmp.h" #include "dtoa.h" #include "fileutils.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* _Py_Mangle is defined in compile.c */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); -#endif - -#ifdef __cplusplus -} -#endif - -/* Argument must be a char or an int in [-128, 127] or [0, 255]. */ -#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) - #include "pyfpe.h" -/* These definitions must match corresponding definitions in graminit.h. - There's code in compile.c that checks that they are the same. */ -#define Py_single_input 256 -#define Py_file_input 257 -#define Py_eval_input 258 - -#ifdef HAVE_PTH -/* GNU pth user-space thread support */ -#include <pth.h> -#endif - -/* Define macros for inline documentation. */ -#define PyDoc_VAR(name) static char name[] -#define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str) -#ifdef WITH_DOC_STRINGS -#define PyDoc_STR(str) str -#else -#define PyDoc_STR(str) "" -#endif - #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index 0fe0956..3946ec5 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -7,6 +7,7 @@ extern "C" { #ifdef PY_SSIZE_T_CLEAN #define PyObject_CallFunction _PyObject_CallFunction_SizeT #define PyObject_CallMethod _PyObject_CallMethod_SizeT +#define _PyObject_CallMethodId _PyObject_CallMethodId_SizeT #endif /* Abstract Object Interface (many thanks to Jim Fulton) */ @@ -307,11 +308,22 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ Python expression: o.method(args). */ + PyAPI_FUNC(PyObject *) _PyObject_CallMethodId(PyObject *o, _Py_Identifier *method, + char *format, ...); + + /* + Like PyObject_CallMethod, but expect a _Py_Identifier* as the + method name. + */ + PyAPI_FUNC(PyObject *) _PyObject_CallFunction_SizeT(PyObject *callable, char *format, ...); PyAPI_FUNC(PyObject *) _PyObject_CallMethod_SizeT(PyObject *o, char *name, char *format, ...); + PyAPI_FUNC(PyObject *) _PyObject_CallMethodId_SizeT(PyObject *o, + _Py_Identifier *name, + char *format, ...); PyAPI_FUNC(PyObject *) PyObject_CallFunctionObjArgs(PyObject *callable, ...); diff --git a/Include/asdl.h b/Include/asdl.h index 9bb0697..42bbbf8 100644 --- a/Include/asdl.h +++ b/Include/asdl.h @@ -3,6 +3,7 @@ typedef PyObject * identifier; typedef PyObject * string; +typedef PyObject * bytes; typedef PyObject * object; /* It would be nice if the code generated by asdl_c.py was completely diff --git a/Include/ast.h b/Include/ast.h index a015336..055e8dc 100644 --- a/Include/ast.h +++ b/Include/ast.h @@ -4,6 +4,7 @@ extern "C" { #endif +PyAPI_FUNC(int) PyAST_Validate(mod_ty); PyAPI_FUNC(mod_ty) PyAST_FromNode( const node *n, PyCompilerFlags *flags, diff --git a/Include/code.h b/Include/code.h index e773b6a..7c7e5bf 100644 --- a/Include/code.h +++ b/Include/code.h @@ -22,6 +22,7 @@ typedef struct { PyObject *co_freevars; /* tuple of strings (free variable names) */ PyObject *co_cellvars; /* tuple of strings (cell variable names) */ /* The rest doesn't count for hash or comparisons */ + unsigned char *co_cell2arg; /* Maps cell vars which are arguments. */ PyObject *co_filename; /* unicode (where it was loaded from) */ PyObject *co_name; /* unicode (name, for reference) */ int co_firstlineno; /* first source line number */ @@ -57,6 +58,11 @@ typedef struct { #define CO_FUTURE_BARRY_AS_BDFL 0x40000 +/* This value is found in the co_cell2arg array when the associated cell + variable does not correspond to an argument. The maximum number of + arguments is 255 (indexed up to 254), so 255 work as a special flag.*/ +#define CO_CELL_NOT_AN_ARG 255 + /* This should be defined if a future statement modifies the syntax. For example, when a keyword is added. */ diff --git a/Include/codecs.h b/Include/codecs.h index dff09e7..0d9e9b4 100644 --- a/Include/codecs.h +++ b/Include/codecs.h @@ -174,6 +174,8 @@ PyAPI_FUNC(PyObject *) PyCodec_XMLCharRefReplaceErrors(PyObject *exc); /* replace the unicode encode error with backslash escapes (\x, \u and \U) */ PyAPI_FUNC(PyObject *) PyCodec_BackslashReplaceErrors(PyObject *exc); +PyAPI_DATA(const char *) Py_hexdigits; + #ifdef __cplusplus } #endif diff --git a/Include/compile.h b/Include/compile.h index bc53b39..ac2636d 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -1,7 +1,7 @@ -#ifndef Py_LIMITED_API #ifndef Py_COMPILE_H #define Py_COMPILE_H +#ifndef Py_LIMITED_API #include "code.h" #ifdef __cplusplus @@ -38,9 +38,19 @@ PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx( PyArena *arena); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); +/* _Py_Mangle is defined in compile.c */ +PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); #ifdef __cplusplus } #endif -#endif /* !Py_COMPILE_H */ + #endif /* !Py_LIMITED_API */ + +/* These definitions must match corresponding definitions in graminit.h. + There's code in compile.c that checks that they are the same. */ +#define Py_single_input 256 +#define Py_file_input 257 +#define Py_eval_input 258 + +#endif /* !Py_COMPILE_H */ diff --git a/Include/complexobject.h b/Include/complexobject.h index c379b08..3e4ecff 100644 --- a/Include/complexobject.h +++ b/Include/complexobject.h @@ -64,8 +64,9 @@ PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op); (Advanced String Formatting). */ #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyComplex_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif #ifdef __cplusplus diff --git a/Include/datetime.h b/Include/datetime.h index db57a18..41e3bcf 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -135,6 +135,12 @@ typedef struct (((PyDateTime_Time*)o)->data[4] << 8) | \ ((PyDateTime_Time*)o)->data[5]) +/* Apply for time delta instances */ +#define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) +#define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) +#define PyDateTime_DELTA_GET_MICROSECONDS(o) \ + (((PyDateTime_Delta*)o)->microseconds) + /* Define structure for C API. */ typedef struct { diff --git a/Include/descrobject.h b/Include/descrobject.h index 646b3cc..e2ba97f 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -42,6 +42,7 @@ typedef struct { PyObject_HEAD PyTypeObject *d_type; PyObject *d_name; + PyObject *d_qualname; } PyDescrObject; #define PyDescr_COMMON PyDescrObject d_common diff --git a/Include/dictobject.h b/Include/dictobject.h index b026785..ed44e20 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -129,6 +129,8 @@ PyAPI_FUNC(int) _PyDict_Contains(PyObject *mp, PyObject *key, Py_hash_t hash); PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused); PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); + +PyAPI_FUNC(int) PyDict_ClearFreeList(void); #endif /* PyDict_Update(mp, other) is equivalent to PyDict_Merge(mp, other, 1). */ diff --git a/Include/errcode.h b/Include/errcode.h index 6bb3cc1..5946686 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -30,6 +30,7 @@ extern "C" { #define E_EOLS 24 /* EOL in single-quoted string */ #define E_LINECONT 25 /* Unexpected characters after a line continuation */ #define E_IDENTIFIER 26 /* Invalid characters in identifier */ +#define E_BADSINGLE 27 /* Ill-formed single statement input */ #ifdef __cplusplus } diff --git a/Include/floatobject.h b/Include/floatobject.h index 90f0a45..0ca4881 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -27,12 +27,12 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) #endif -#define Py_RETURN_INF(sign) do \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ - } else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ - } while(0) +#define Py_RETURN_INF(sign) do \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(Py_HUGE_VAL); \ + } else { \ + return PyFloat_FromDouble(-Py_HUGE_VAL); \ + } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); PyAPI_FUNC(double) PyFloat_GetMin(void); @@ -113,8 +113,9 @@ PyAPI_FUNC(int) PyFloat_ClearFreeList(void); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Include/frameobject.h b/Include/frameobject.h index 1fb64bb..55447b7 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -9,45 +9,46 @@ extern "C" { #endif typedef struct { - int b_type; /* what kind of block this is */ - int b_handler; /* where to jump to find handler */ - int b_level; /* value stack level to pop to */ + int b_type; /* what kind of block this is */ + int b_handler; /* where to jump to find handler */ + int b_level; /* value stack level to pop to */ } PyTryBlock; typedef struct _frame { PyObject_VAR_HEAD - struct _frame *f_back; /* previous frame, or NULL */ - PyCodeObject *f_code; /* code segment */ - PyObject *f_builtins; /* builtin symbol table (PyDictObject) */ - PyObject *f_globals; /* global symbol table (PyDictObject) */ - PyObject *f_locals; /* local symbol table (any mapping) */ - PyObject **f_valuestack; /* points after the last local */ + struct _frame *f_back; /* previous frame, or NULL */ + PyCodeObject *f_code; /* code segment */ + PyObject *f_builtins; /* builtin symbol table (PyDictObject) */ + PyObject *f_globals; /* global symbol table (PyDictObject) */ + PyObject *f_locals; /* local symbol table (any mapping) */ + PyObject **f_valuestack; /* points after the last local */ /* Next free slot in f_valuestack. Frame creation sets to f_valuestack. Frame evaluation usually NULLs it, but a frame that yields sets it to the current stack top. */ PyObject **f_stacktop; - PyObject *f_trace; /* Trace function */ - - /* In a generator, we need to be able to swap between the exception - state inside the generator and the exception state of the calling - frame (which shouldn't be impacted when the generator "yields" - from an except handler). - These three fields exist exactly for that, and are unused for - non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE - macros in ceval.c for details of their use. */ + PyObject *f_trace; /* Trace function */ + PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */ + + /* In a generator, we need to be able to swap between the exception + state inside the generator and the exception state of the calling + frame (which shouldn't be impacted when the generator "yields" + from an except handler). + These three fields exist exactly for that, and are unused for + non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE + macros in ceval.c for details of their use. */ PyObject *f_exc_type, *f_exc_value, *f_exc_traceback; PyThreadState *f_tstate; - int f_lasti; /* Last instruction if called */ + int f_lasti; /* Last instruction if called */ /* Call PyFrame_GetLineNumber() instead of reading this field directly. As of 2.3 f_lineno is only valid when tracing is active (i.e. when f_trace is set). At other times we use PyCode_Addr2Line to calculate the line from the current bytecode index. */ - int f_lineno; /* Current line number */ - int f_iblock; /* index in f_blockstack */ + int f_lineno; /* Current line number */ + int f_iblock; /* index in f_blockstack */ PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ - PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ + PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */ } PyFrameObject; diff --git a/Include/funcobject.h b/Include/funcobject.h index 521d87b..cc1426c 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -31,6 +31,7 @@ typedef struct { PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_qualname; /* The qualified name */ /* Invariant: * func_closure contains the bindings for func_code->co_freevars, so @@ -44,6 +45,7 @@ PyAPI_DATA(PyTypeObject) PyFunction_Type; #define PyFunction_Check(op) (Py_TYPE(op) == &PyFunction_Type) PyAPI_FUNC(PyObject *) PyFunction_New(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyFunction_NewWithQualName(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetCode(PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetGlobals(PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetModule(PyObject *); diff --git a/Include/genobject.h b/Include/genobject.h index d29fb1e..e7a1c01 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -11,20 +11,20 @@ extern "C" { struct _frame; /* Avoid including frameobject.h */ typedef struct { - PyObject_HEAD - /* The gi_ prefix is intended to remind of generator-iterator. */ + PyObject_HEAD + /* The gi_ prefix is intended to remind of generator-iterator. */ - /* Note: gi_frame can be NULL if the generator is "finished" */ - struct _frame *gi_frame; + /* Note: gi_frame can be NULL if the generator is "finished" */ + struct _frame *gi_frame; - /* True if generator is being executed. */ - int gi_running; + /* True if generator is being executed. */ + int gi_running; - /* The code object backing the generator */ - PyObject *gi_code; + /* The code object backing the generator */ + PyObject *gi_code; - /* List of weak reference. */ - PyObject *gi_weakreflist; + /* List of weak reference. */ + PyObject *gi_weakreflist; } PyGenObject; PyAPI_DATA(PyTypeObject) PyGen_Type; @@ -34,6 +34,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *); +PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **); #ifdef __cplusplus } diff --git a/Include/graminit.h b/Include/graminit.h index e0e27f9..3ec949a 100644 --- a/Include/graminit.h +++ b/Include/graminit.h @@ -81,3 +81,4 @@ #define comp_if 334 #define encoding_decl 335 #define yield_expr 336 +#define yield_arg 337 diff --git a/Include/import.h b/Include/import.h index 400e97c..45544111 100644 --- a/Include/import.h +++ b/Include/import.h @@ -24,7 +24,16 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleWithPathnames( char *pathname, /* decoded from the filesystem encoding */ char *cpathname /* decoded from the filesystem encoding */ ); +PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject( + PyObject *name, + PyObject *co, + PyObject *pathname, + PyObject *cpathname + ); PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void); +PyAPI_FUNC(PyObject *) PyImport_AddModuleObject( + PyObject *name + ); PyAPI_FUNC(PyObject *) PyImport_AddModule( const char *name /* UTF-8 encoded string */ ); @@ -35,7 +44,14 @@ PyAPI_FUNC(PyObject *) PyImport_ImportModuleNoBlock( const char *name /* UTF-8 encoded string */ ); PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevel( - char *name, /* UTF-8 encoded string */ + const char *name, /* UTF-8 encoded string */ + PyObject *globals, + PyObject *locals, + PyObject *fromlist, + int level + ); +PyAPI_FUNC(PyObject *) PyImport_ImportModuleLevelObject( + PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, @@ -49,6 +65,9 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path); PyAPI_FUNC(PyObject *) PyImport_Import(PyObject *name); PyAPI_FUNC(PyObject *) PyImport_ReloadModule(PyObject *m); PyAPI_FUNC(void) PyImport_Cleanup(void); +PyAPI_FUNC(int) PyImport_ImportFrozenModuleObject( + PyObject *name + ); PyAPI_FUNC(int) PyImport_ImportFrozenModule( char *name /* UTF-8 encoded string */ ); @@ -65,17 +84,17 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void); PyAPI_FUNC(void) _PyImport_ReInitLock(void); PyAPI_FUNC(PyObject *)_PyImport_FindBuiltin( - char *name /* UTF-8 encoded string */ + const char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(PyObject *)_PyImport_FindExtensionUnicode(char *, PyObject *); +PyAPI_FUNC(PyObject *)_PyImport_FindExtensionObject(PyObject *, PyObject *); PyAPI_FUNC(int)_PyImport_FixupBuiltin( PyObject *mod, char *name /* UTF-8 encoded string */ ); -PyAPI_FUNC(int)_PyImport_FixupExtensionUnicode(PyObject*, char *, PyObject *); +PyAPI_FUNC(int)_PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *); struct _inittab { - char *name; + char *name; /* ASCII encoded string */ PyObject* (*initfunc)(void); }; PyAPI_DATA(struct _inittab *) PyImport_Inittab; diff --git a/Include/intrcheck.h b/Include/intrcheck.h index 3b67ed0..f53fee1 100644 --- a/Include/intrcheck.h +++ b/Include/intrcheck.h @@ -8,6 +8,12 @@ extern "C" { PyAPI_FUNC(int) PyOS_InterruptOccurred(void); PyAPI_FUNC(void) PyOS_InitInterrupts(void); PyAPI_FUNC(void) PyOS_AfterFork(void); +PyAPI_FUNC(int) _PyOS_IsMainThread(void); + +#ifdef MS_WINDOWS +/* windows.h is not included by Python.h so use void* instead of HANDLE */ +PyAPI_FUNC(void*) _PyOS_SigintEvent(void); +#endif #ifdef __cplusplus } diff --git a/Include/listobject.h b/Include/listobject.h index 949b1a3..6fd374b 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -62,6 +62,8 @@ PyAPI_FUNC(int) PyList_Reverse(PyObject *); PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); + +PyAPI_FUNC(int) PyList_ClearFreeList(void); #endif /* Macro, trading safety for speed */ diff --git a/Include/longobject.h b/Include/longobject.h index c09565a..c58ddf4 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -12,7 +12,7 @@ typedef struct _longobject PyLongObject; /* Revealed in longintrepr.h */ PyAPI_DATA(PyTypeObject) PyLong_Type; #define PyLong_Check(op) \ - PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) + PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LONG_SUBCLASS) #define PyLong_CheckExact(op) (Py_TYPE(op) == &PyLong_Type) PyAPI_FUNC(PyObject *) PyLong_FromLong(long); @@ -80,6 +80,7 @@ PyAPI_FUNC(PY_LONG_LONG) PyLong_AsLongLongAndOverflow(PyObject *, int *); PyAPI_FUNC(PyObject *) PyLong_FromString(char *, char **, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyLong_FromUnicode(Py_UNICODE*, Py_ssize_t, int); +PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); #endif #ifndef Py_LIMITED_API @@ -121,8 +122,8 @@ PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); enough memory to create the Python long. */ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); /* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long v to a base-256 integer, stored in array bytes. Normally return 0, @@ -144,8 +145,8 @@ PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( case, but bytes holds the least-signficant n bytes of the true value. */ PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); + unsigned char* bytes, size_t n, + int little_endian, int is_signed); /* _PyLong_Format: Convert the long to a string object with given base, @@ -155,8 +156,9 @@ PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *aa, int base); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyLong_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif /* Py_LIMITED_API */ /* These aren't really part of the long object, but they're handy. The diff --git a/Include/memoryobject.h b/Include/memoryobject.h index 62ecbd6..aff5d99 100644 --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -69,6 +69,7 @@ PyAPI_FUNC(PyObject *) PyMemoryView_FromBuffer(Py_buffer *info); typedef struct { PyObject_HEAD Py_buffer view; + Py_hash_t hash; } PyMemoryViewObject; #endif diff --git a/Include/methodobject.h b/Include/methodobject.h index 7e67c0b..d798d13 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -30,7 +30,8 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *); #define PyCFunction_GET_FUNCTION(func) \ (((PyCFunctionObject *)func) -> m_ml -> ml_meth) #define PyCFunction_GET_SELF(func) \ - (((PyCFunctionObject *)func) -> m_self) + (((PyCFunctionObject *)func) -> m_ml -> ml_flags & METH_STATIC ? \ + NULL : ((PyCFunctionObject *)func) -> m_self) #define PyCFunction_GET_FLAGS(func) \ (((PyCFunctionObject *)func) -> m_ml -> ml_flags) #endif diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 7b2bf1c..8013dd9 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -12,10 +12,14 @@ PyAPI_DATA(PyTypeObject) PyModule_Type; #define PyModule_Check(op) PyObject_TypeCheck(op, &PyModule_Type) #define PyModule_CheckExact(op) (Py_TYPE(op) == &PyModule_Type) +PyAPI_FUNC(PyObject *) PyModule_NewObject( + PyObject *name + ); PyAPI_FUNC(PyObject *) PyModule_New( const char *name /* UTF-8 encoded string */ ); PyAPI_FUNC(PyObject *) PyModule_GetDict(PyObject *); +PyAPI_FUNC(PyObject *) PyModule_GetNameObject(PyObject *); PyAPI_FUNC(const char *) PyModule_GetName(PyObject *); PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *); PyAPI_FUNC(PyObject *) PyModule_GetFilenameObject(PyObject *); diff --git a/Include/node.h b/Include/node.h index e23e709..d161195 100644 --- a/Include/node.h +++ b/Include/node.h @@ -28,6 +28,7 @@ PyAPI_FUNC(void) PyNode_Free(node *n); #define RCHILD(n, i) (CHILD(n, NCH(n) + i)) #define TYPE(n) ((n)->n_type) #define STR(n) ((n)->n_str) +#define LINENO(n) ((n)->n_lineno) /* Assert that the type of a node is what we expect */ #define REQ(n, type) assert(TYPE(n) == (type)) diff --git a/Include/object.h b/Include/object.h index 2528841..6907a44 100644 --- a/Include/object.h +++ b/Include/object.h @@ -117,6 +117,35 @@ typedef struct { #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +/********************* String Literals ****************************************/ +/* This structure helps managing static strings. The basic usage goes like this: + Instead of doing + + r = PyObject_CallMethod(o, "foo", "args", ...); + + do + + _Py_IDENTIFIER(foo); + ... + r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...); + + PyId_foo is a static variable, either on block level or file level. On first + usage, the string "foo" is interned, and the structures are linked. On interpreter + shutdown, all strings are released (through _PyUnicode_ClearStaticStrings). + + Alternatively, _Py_static_string allows to choose the variable name. + _PyUnicode_FromId returns a borrowed reference to the interned string. + _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. +*/ +typedef struct _Py_Identifier { + struct _Py_Identifier *next; + const char* string; + PyObject *object; +} _Py_Identifier; + +#define _Py_static_string(varname, value) static _Py_Identifier varname = { 0, value, 0 } +#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) + /* Type objects contain a string containing the type name (to help somewhat in debugging), the allocation parameters (see PyObject_New() and @@ -418,7 +447,7 @@ typedef struct _heaptypeobject { a given operator (e.g. __getitem__). see add_operators() in typeobject.c . */ PyBufferProcs as_buffer; - PyObject *ht_name, *ht_slots; + PyObject *ht_name, *ht_slots, *ht_qualname; /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; @@ -448,13 +477,14 @@ PyAPI_FUNC(PyObject *) PyType_GenericNew(PyTypeObject *, PyObject *, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, char *, PyObject **); +PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); #endif PyAPI_FUNC(unsigned int) PyType_ClearCache(void); PyAPI_FUNC(void) PyType_Modified(PyTypeObject *); /* Generic operations on objects */ +struct _Py_Identifier; #ifndef Py_LIMITED_API PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); @@ -472,6 +502,10 @@ PyAPI_FUNC(int) PyObject_HasAttrString(PyObject *, const char *); PyAPI_FUNC(PyObject *) PyObject_GetAttr(PyObject *, PyObject *); PyAPI_FUNC(int) PyObject_SetAttr(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(int) PyObject_HasAttr(PyObject *, PyObject *); +PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); +PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, struct _Py_Identifier *); +PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, struct _Py_Identifier *, PyObject *); +PyAPI_FUNC(int) _PyObject_HasAttrId(PyObject *, struct _Py_Identifier *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); #endif @@ -515,6 +549,7 @@ PyAPI_FUNC(void) Py_ReprLeave(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double); PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*); +PyAPI_FUNC(Py_hash_t) _Py_HashBytes(unsigned char*, Py_ssize_t); #endif /* Helper for passing objects to printf and the like */ @@ -793,6 +828,10 @@ not implemented for a given type combination. PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) +/* Macro for returning Py_NotImplemented from a function */ +#define Py_RETURN_NOTIMPLEMENTED \ + return Py_INCREF(Py_NotImplemented), Py_NotImplemented + /* Rich comparison opcodes */ #define Py_LT 0 #define Py_LE 1 diff --git a/Include/opcode.h b/Include/opcode.h index 6b10944..a90184d 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -7,117 +7,117 @@ extern "C" { /* Instruction opcodes for compiled code */ -#define STOP_CODE 0 -#define POP_TOP 1 -#define ROT_TWO 2 -#define ROT_THREE 3 -#define DUP_TOP 4 +#define POP_TOP 1 +#define ROT_TWO 2 +#define ROT_THREE 3 +#define DUP_TOP 4 #define DUP_TOP_TWO 5 -#define NOP 9 +#define NOP 9 -#define UNARY_POSITIVE 10 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 -#define UNARY_INVERT 15 +#define UNARY_INVERT 15 -#define BINARY_POWER 19 +#define BINARY_POWER 19 -#define BINARY_MULTIPLY 20 +#define BINARY_MULTIPLY 20 -#define BINARY_MODULO 22 -#define BINARY_ADD 23 -#define BINARY_SUBTRACT 24 -#define BINARY_SUBSCR 25 +#define BINARY_MODULO 22 +#define BINARY_ADD 23 +#define BINARY_SUBTRACT 24 +#define BINARY_SUBSCR 25 #define BINARY_FLOOR_DIVIDE 26 #define BINARY_TRUE_DIVIDE 27 #define INPLACE_FLOOR_DIVIDE 28 #define INPLACE_TRUE_DIVIDE 29 -#define STORE_MAP 54 -#define INPLACE_ADD 55 -#define INPLACE_SUBTRACT 56 -#define INPLACE_MULTIPLY 57 - -#define INPLACE_MODULO 59 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 - -#define BINARY_LSHIFT 62 -#define BINARY_RSHIFT 63 -#define BINARY_AND 64 -#define BINARY_XOR 65 -#define BINARY_OR 66 -#define INPLACE_POWER 67 -#define GET_ITER 68 -#define STORE_LOCALS 69 -#define PRINT_EXPR 70 +#define STORE_MAP 54 +#define INPLACE_ADD 55 +#define INPLACE_SUBTRACT 56 +#define INPLACE_MULTIPLY 57 + +#define INPLACE_MODULO 59 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 + +#define BINARY_LSHIFT 62 +#define BINARY_RSHIFT 63 +#define BINARY_AND 64 +#define BINARY_XOR 65 +#define BINARY_OR 66 +#define INPLACE_POWER 67 +#define GET_ITER 68 +#define STORE_LOCALS 69 +#define PRINT_EXPR 70 #define LOAD_BUILD_CLASS 71 - -#define INPLACE_LSHIFT 75 -#define INPLACE_RSHIFT 76 -#define INPLACE_AND 77 -#define INPLACE_XOR 78 -#define INPLACE_OR 79 -#define BREAK_LOOP 80 +#define YIELD_FROM 72 + +#define INPLACE_LSHIFT 75 +#define INPLACE_RSHIFT 76 +#define INPLACE_AND 77 +#define INPLACE_XOR 78 +#define INPLACE_OR 79 +#define BREAK_LOOP 80 #define WITH_CLEANUP 81 -#define RETURN_VALUE 83 -#define IMPORT_STAR 84 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 -#define YIELD_VALUE 86 -#define POP_BLOCK 87 -#define END_FINALLY 88 -#define POP_EXCEPT 89 +#define YIELD_VALUE 86 +#define POP_BLOCK 87 +#define END_FINALLY 88 +#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */ +#define HAVE_ARGUMENT 90 /* Opcodes from here have an argument: */ -#define STORE_NAME 90 /* Index in name list */ -#define DELETE_NAME 91 /* "" */ -#define UNPACK_SEQUENCE 92 /* Number of sequence items */ -#define FOR_ITER 93 +#define STORE_NAME 90 /* Index in name list */ +#define DELETE_NAME 91 /* "" */ +#define UNPACK_SEQUENCE 92 /* Number of sequence items */ +#define FOR_ITER 93 #define UNPACK_EX 94 /* Num items before variable part + (Num items after variable part << 8) */ -#define STORE_ATTR 95 /* Index in name list */ -#define DELETE_ATTR 96 /* "" */ -#define STORE_GLOBAL 97 /* "" */ -#define DELETE_GLOBAL 98 /* "" */ - -#define LOAD_CONST 100 /* Index in const list */ -#define LOAD_NAME 101 /* Index in name list */ -#define BUILD_TUPLE 102 /* Number of tuple items */ -#define BUILD_LIST 103 /* Number of list items */ -#define BUILD_SET 104 /* Number of set items */ -#define BUILD_MAP 105 /* Always zero for now */ -#define LOAD_ATTR 106 /* Index in name list */ -#define COMPARE_OP 107 /* Comparison operator */ -#define IMPORT_NAME 108 /* Index in name list */ -#define IMPORT_FROM 109 /* Index in name list */ - -#define JUMP_FORWARD 110 /* Number of bytes to skip */ -#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */ -#define JUMP_IF_TRUE_OR_POP 112 /* "" */ -#define JUMP_ABSOLUTE 113 /* "" */ -#define POP_JUMP_IF_FALSE 114 /* "" */ -#define POP_JUMP_IF_TRUE 115 /* "" */ - -#define LOAD_GLOBAL 116 /* Index in name list */ - -#define CONTINUE_LOOP 119 /* Start of loop (absolute) */ -#define SETUP_LOOP 120 /* Target address (relative) */ -#define SETUP_EXCEPT 121 /* "" */ -#define SETUP_FINALLY 122 /* "" */ - -#define LOAD_FAST 124 /* Local variable number */ -#define STORE_FAST 125 /* Local variable number */ -#define DELETE_FAST 126 /* Local variable number */ - -#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */ +#define STORE_ATTR 95 /* Index in name list */ +#define DELETE_ATTR 96 /* "" */ +#define STORE_GLOBAL 97 /* "" */ +#define DELETE_GLOBAL 98 /* "" */ + +#define LOAD_CONST 100 /* Index in const list */ +#define LOAD_NAME 101 /* Index in name list */ +#define BUILD_TUPLE 102 /* Number of tuple items */ +#define BUILD_LIST 103 /* Number of list items */ +#define BUILD_SET 104 /* Number of set items */ +#define BUILD_MAP 105 /* Always zero for now */ +#define LOAD_ATTR 106 /* Index in name list */ +#define COMPARE_OP 107 /* Comparison operator */ +#define IMPORT_NAME 108 /* Index in name list */ +#define IMPORT_FROM 109 /* Index in name list */ + +#define JUMP_FORWARD 110 /* Number of bytes to skip */ +#define JUMP_IF_FALSE_OR_POP 111 /* Target byte offset from beginning of code */ +#define JUMP_IF_TRUE_OR_POP 112 /* "" */ +#define JUMP_ABSOLUTE 113 /* "" */ +#define POP_JUMP_IF_FALSE 114 /* "" */ +#define POP_JUMP_IF_TRUE 115 /* "" */ + +#define LOAD_GLOBAL 116 /* Index in name list */ + +#define CONTINUE_LOOP 119 /* Start of loop (absolute) */ +#define SETUP_LOOP 120 /* Target address (relative) */ +#define SETUP_EXCEPT 121 /* "" */ +#define SETUP_FINALLY 122 /* "" */ + +#define LOAD_FAST 124 /* Local variable number */ +#define STORE_FAST 125 /* Local variable number */ +#define DELETE_FAST 126 /* Local variable number */ + +#define RAISE_VARARGS 130 /* Number of raise arguments (1, 2 or 3) */ /* CALL_FUNCTION_XXX opcodes defined below depend on this definition */ -#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */ -#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */ -#define BUILD_SLICE 133 /* Number of items */ +#define CALL_FUNCTION 131 /* #args + (#kwargs<<8) */ +#define MAKE_FUNCTION 132 /* #defaults + #kwdefaults<<8 + #annotations<<16 */ +#define BUILD_SLICE 133 /* Number of items */ #define MAKE_CLOSURE 134 /* same as MAKE_FUNCTION */ #define LOAD_CLOSURE 135 /* Load free variable from closure */ @@ -127,9 +127,9 @@ extern "C" { /* The next 3 opcodes must be contiguous and satisfy (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1 */ -#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */ -#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */ -#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */ +#define CALL_FUNCTION_VAR 140 /* #args + (#kwargs<<8) */ +#define CALL_FUNCTION_KW 141 /* #args + (#kwargs<<8) */ +#define CALL_FUNCTION_VAR_KW 142 /* #args + (#kwargs<<8) */ #define SETUP_WITH 143 @@ -149,7 +149,7 @@ extern "C" { enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE, - PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD}; + PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD}; #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) diff --git a/Include/parsetok.h b/Include/parsetok.h index 4b7694f..911dfc1 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -9,7 +9,10 @@ extern "C" { typedef struct { int error; - const char *filename; /* decoded from the filesystem encoding */ +#ifndef PGEN + /* The filename is useless for pgen, see comment in tok_state structure */ + PyObject *filename; +#endif int lineno; int offset; char *text; /* UTF-8-encoded string */ @@ -66,8 +69,10 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx( perrdetail *err_ret, int *flags); -/* Note that he following function is defined in pythonrun.c not parsetok.c. */ +/* Note that the following functions are defined in pythonrun.c, + not in parsetok.c */ PyAPI_FUNC(void) PyParser_SetError(perrdetail *); +PyAPI_FUNC(void) PyParser_ClearError(perrdetail *); #ifdef __cplusplus } diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 1b8e6d4..4996e8e 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,19 +17,15 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 2 -#define PY_MICRO_VERSION 2 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_MINOR_VERSION 3 +#define PY_MICRO_VERSION 0 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.2.2+" +#define PY_VERSION "3.3.0a0" /*--end constants--*/ -/* Subversion Revision number of this file (not of the repository). Empty - since Mercurial migration. */ -#define PY_PATCHLEVEL_REVISION "" - /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */ #define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \ diff --git a/Include/py_curses.h b/Include/py_curses.h index a891c42..f2c08f6 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -76,6 +76,7 @@ extern "C" { typedef struct { PyObject_HEAD WINDOW *win; + char *encoding; } PyCursesWindowObject; #define PyCursesWindow_Check(v) (Py_TYPE(v) == &PyCursesWindow_Type) diff --git a/Include/pydebug.h b/Include/pydebug.h index 70c88f6..7173fe3 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -16,7 +16,6 @@ PyAPI_DATA(int) Py_BytesWarningFlag; PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; -PyAPI_DATA(int) Py_DivisionWarningFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; PyAPI_DATA(int) Py_NoUserSiteDirectory; PyAPI_DATA(int) Py_UnbufferedStdioFlag; @@ -26,8 +25,6 @@ PyAPI_DATA(int) Py_UnbufferedStdioFlag; PYTHONPATH and PYTHONHOME from the environment */ #define Py_GETENV(s) (Py_IgnoreEnvironmentFlag ? NULL : getenv(s)) -PyAPI_FUNC(void) Py_FatalError(const char *message); - #ifdef __cplusplus } #endif diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 4bb3c01..1bd0442 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -45,18 +45,23 @@ typedef struct { PyObject *myerrno; PyObject *strerror; PyObject *filename; -} PyEnvironmentErrorObject; - #ifdef MS_WINDOWS -typedef struct { - PyException_HEAD - PyObject *myerrno; - PyObject *strerror; - PyObject *filename; PyObject *winerror; -} PyWindowsErrorObject; #endif + Py_ssize_t written; /* only for BlockingIOError, -1 otherwise */ +} PyOSErrorObject; + +typedef struct { + PyException_HEAD + PyObject *value; +} PyStopIterationObject; + +/* Compatibility typedefs */ +typedef PyOSErrorObject PyEnvironmentErrorObject; +#ifdef MS_WINDOWS +typedef PyOSErrorObject PyWindowsErrorObject; #endif +#endif /* !Py_LIMITED_API */ /* Error handling definitions */ @@ -70,7 +75,17 @@ PyAPI_FUNC(PyObject *) PyErr_Occurred(void); PyAPI_FUNC(void) PyErr_Clear(void); PyAPI_FUNC(void) PyErr_Fetch(PyObject **, PyObject **, PyObject **); PyAPI_FUNC(void) PyErr_Restore(PyObject *, PyObject *, PyObject *); -PyAPI_FUNC(void) Py_FatalError(const char *message); + +#if defined(__clang__) || \ + (defined(__GNUC__) && \ + ((__GNUC_MAJOR__ >= 3) || \ + (__GNUC_MAJOR__ == 2) && (__GNUC_MINOR__ >= 5))) +#define _Py_NO_RETURN __attribute__((__noreturn__)) +#else +#define _Py_NO_RETURN +#endif + +PyAPI_FUNC(void) Py_FatalError(const char *message) _Py_NO_RETURN; #if defined(Py_DEBUG) || defined(Py_LIMITED_API) #define _PyErr_OCCURRED() PyErr_Occurred() @@ -122,10 +137,9 @@ PyAPI_DATA(PyObject *) PyExc_LookupError; PyAPI_DATA(PyObject *) PyExc_AssertionError; PyAPI_DATA(PyObject *) PyExc_AttributeError; +PyAPI_DATA(PyObject *) PyExc_BufferError; PyAPI_DATA(PyObject *) PyExc_EOFError; PyAPI_DATA(PyObject *) PyExc_FloatingPointError; -PyAPI_DATA(PyObject *) PyExc_EnvironmentError; -PyAPI_DATA(PyObject *) PyExc_IOError; PyAPI_DATA(PyObject *) PyExc_OSError; PyAPI_DATA(PyObject *) PyExc_ImportError; PyAPI_DATA(PyObject *) PyExc_IndexError; @@ -150,6 +164,27 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError; PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError; PyAPI_DATA(PyObject *) PyExc_ValueError; PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError; + +PyAPI_DATA(PyObject *) PyExc_BlockingIOError; +PyAPI_DATA(PyObject *) PyExc_BrokenPipeError; +PyAPI_DATA(PyObject *) PyExc_ChildProcessError; +PyAPI_DATA(PyObject *) PyExc_ConnectionError; +PyAPI_DATA(PyObject *) PyExc_ConnectionAbortedError; +PyAPI_DATA(PyObject *) PyExc_ConnectionRefusedError; +PyAPI_DATA(PyObject *) PyExc_ConnectionResetError; +PyAPI_DATA(PyObject *) PyExc_FileExistsError; +PyAPI_DATA(PyObject *) PyExc_FileNotFoundError; +PyAPI_DATA(PyObject *) PyExc_InterruptedError; +PyAPI_DATA(PyObject *) PyExc_IsADirectoryError; +PyAPI_DATA(PyObject *) PyExc_NotADirectoryError; +PyAPI_DATA(PyObject *) PyExc_PermissionError; +PyAPI_DATA(PyObject *) PyExc_ProcessLookupError; +PyAPI_DATA(PyObject *) PyExc_TimeoutError; + + +/* Compatibility aliases */ +PyAPI_DATA(PyObject *) PyExc_EnvironmentError; +PyAPI_DATA(PyObject *) PyExc_IOError; #ifdef MS_WINDOWS PyAPI_DATA(PyObject *) PyExc_WindowsError; #endif @@ -157,8 +192,6 @@ PyAPI_DATA(PyObject *) PyExc_WindowsError; PyAPI_DATA(PyObject *) PyExc_VMSError; #endif -PyAPI_DATA(PyObject *) PyExc_BufferError; - PyAPI_DATA(PyObject *) PyExc_RecursionErrorInst; /* Predefined warning categories */ @@ -198,8 +231,6 @@ PyAPI_FUNC(PyObject *) PyErr_Format( ); #ifdef MS_WINDOWS -PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilenameObject( - int, const char *); PyAPI_FUNC(PyObject *) PyErr_SetFromWindowsErrWithFilename( int ierr, const char *filename /* decoded from the filesystem encoding */ @@ -293,6 +324,12 @@ PyAPI_FUNC(PyObject *) PyUnicodeTranslateError_Create( Py_ssize_t end, const char *reason /* UTF-8 encoded string */ ); +PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( + PyObject *object, + Py_ssize_t start, + Py_ssize_t end, + const char *reason /* UTF-8 encoded string */ + ); #endif /* get the encoding attribute */ @@ -348,6 +385,8 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason( const char *reason /* UTF-8 encoded string */ ); +/* create a StopIteration exception with the given value */ +PyAPI_FUNC(PyObject *) PyStopIteration_Create(PyObject *); /* These APIs aren't really part of the error implementation, but often needed to format error messages; the native C lib APIs are diff --git a/Include/pymacro.h b/Include/pymacro.h new file mode 100644 index 0000000..1dc0c61 --- /dev/null +++ b/Include/pymacro.h @@ -0,0 +1,55 @@ +#ifndef Py_PYMACRO_H +#define Py_PYMACRO_H + +#define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) +#define Py_MAX(x, y) (((x) > (y)) ? (x) : (y)) + +/* Argument must be a char or an int in [-128, 127] or [0, 255]. */ +#define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) + + +/* Assert a build-time dependency, as an expression. + + Your compile will fail if the condition isn't true, or can't be evaluated + by the compiler. This can be used in an expression: its value is 0. + + Example: + + #define foo_to_char(foo) \ + ((char *)(foo) \ + + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + Written by Rusty Russell, public domain, http://ccodearchive.net/ */ +#define Py_BUILD_ASSERT_EXPR(cond) \ + (sizeof(char [1 - 2*!(cond)]) - 1) + +/* Get the number of elements in a visible array + + This does not work on pointers, or arrays declared as [], or function + parameters. With correct compiler support, such usage will cause a build + error (see Py_BUILD_ASSERT_EXPR). + + Written by Rusty Russell, public domain, http://ccodearchive.net/ */ +#if defined(__GNUC__) +/* Two gcc extensions. + &a[0] degrades to a pointer: a different type from an array */ +#define Py_ARRAY_LENGTH(array) \ + (sizeof(array) / sizeof((array)[0]) \ + + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \ + typeof(&(array)[0])))) +#else +#define Py_ARRAY_LENGTH(array) \ + (sizeof(array) / sizeof((array)[0])) +#endif + + +/* Define macros for inline documentation. */ +#define PyDoc_VAR(name) static char name[] +#define PyDoc_STRVAR(name,str) PyDoc_VAR(name) = PyDoc_STR(str) +#ifdef WITH_DOC_STRINGS +#define PyDoc_STR(str) str +#else +#define PyDoc_STR(str) "" +#endif + +#endif /* Py_PYMACRO_H */ diff --git a/Include/pymath.h b/Include/pymath.h index b4eda66..62a6c42 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -37,12 +37,6 @@ extern double pow(double, double); #endif /* __STDC__ */ #endif /* _MSC_VER */ -#ifdef _OSF_SOURCE -/* OSF1 5.1 doesn't make these available with XOPEN_SOURCE_EXTENDED defined */ -extern int finite(double); -extern double copysign(double, double); -#endif - /* High precision defintion of pi and e (Euler) * The values are taken from libc6's math.h. */ diff --git a/Include/pyport.h b/Include/pyport.h index 269ba68..1fd4dcc 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -135,7 +135,7 @@ Used in: PY_LONG_LONG #define _PyHASH_MULTIPLIER 1000003 /* 0xf4243 */ /* Parameters used for the numeric hash implementation. See notes for - _PyHash_Double in Objects/object.c. Numeric hashes are based on + _Py_HashDouble in Objects/object.c. Numeric hashes are based on reduction modulo the prime 2**_PyHASH_BITS - 1. */ #if SIZEOF_VOID_P >= 8 diff --git a/Include/pystate.h b/Include/pystate.h index b5fe1ad..1bbb4e2 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -74,9 +74,9 @@ typedef struct _ts { struct _frame *frame; int recursion_depth; char overflowed; /* The stack has overflowed. Allow 50 more calls - to handle the runtime error. */ - char recursion_critical; /* The current calls must not cause - a stack overflow. */ + to handle the runtime error. */ + char recursion_critical; /* The current calls must not cause + a stack overflow. */ /* 'tracing' keeps track of the execution depth when tracing/profiling. This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ @@ -160,6 +160,8 @@ typedef enum {PyGILState_LOCKED, PyGILState_UNLOCKED} PyGILState_STATE; +#ifdef WITH_THREAD + /* Ensure that the current thread is ready to call the Python C API, regardless of the current state of Python, or of its thread lock. This may be called as many times as desired @@ -201,6 +203,8 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); */ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +#endif /* #ifdef WITH_THREAD */ + /* The implementation of sys._current_frames() Returns a dict mapping thread id to that thread's current frame. */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 00b4972..fc6c854 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -179,9 +179,6 @@ PyAPI_FUNC(const char *) Py_GetCopyright(void); PyAPI_FUNC(const char *) Py_GetCompiler(void); PyAPI_FUNC(const char *) Py_GetBuildInfo(void); #ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_svnversion(void); -PyAPI_FUNC(const char *) Py_SubversionRevision(void); -PyAPI_FUNC(const char *) Py_SubversionShortBranch(void); PyAPI_FUNC(const char *) _Py_hgidentifier(void); PyAPI_FUNC(const char *) _Py_hgversion(void); #endif @@ -214,6 +211,7 @@ PyAPI_FUNC(void) PyByteArray_Fini(void); PyAPI_FUNC(void) PyFloat_Fini(void); PyAPI_FUNC(void) PyOS_FiniInterrupts(void); PyAPI_FUNC(void) _PyGC_Fini(void); +PyAPI_FUNC(void) PySlice_Fini(void); PyAPI_DATA(PyThreadState *) _Py_Finalizing; #endif diff --git a/Include/pythread.h b/Include/pythread.h index 9806c61..6e9f303 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -32,7 +32,7 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); on a lock (see PyThread_acquire_lock_timed() below). PY_TIMEOUT_MAX is the highest usable value (in microseconds) of that type, and depends on the system threading API. - + NOTE: this isn't the same value as `_thread.TIMEOUT_MAX`. The _thread module exposes a higher-level API, with timeouts expressed in seconds and floating-point numbers allowed. @@ -74,6 +74,8 @@ PyAPI_FUNC(void) PyThread_release_lock(PyThread_type_lock); PyAPI_FUNC(size_t) PyThread_get_stacksize(void); PyAPI_FUNC(int) PyThread_set_stacksize(size_t); +PyAPI_FUNC(PyObject*) PyThread_GetInfo(void); + /* Thread Local Storage (TLS) API */ PyAPI_FUNC(int) PyThread_create_key(void); PyAPI_FUNC(void) PyThread_delete_key(int); diff --git a/Include/setobject.h b/Include/setobject.h index 6234111..00e5344 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -99,6 +99,8 @@ PyAPI_FUNC(int) _PySet_NextEntry(PyObject *set, Py_ssize_t *pos, PyObject **key, PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); + +PyAPI_FUNC(int) PySet_ClearFreeList(void); #endif #ifdef __cplusplus diff --git a/Include/symtable.h b/Include/symtable.h index fd7de04..82f6269 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -23,10 +23,13 @@ struct symtable { PyObject *st_blocks; /* dict: map AST node addresses * to symbol table entries */ PyObject *st_stack; /* list: stack of namespace info */ - PyObject *st_global; /* borrowed ref to st_top->st_symbols */ - int st_nblocks; /* number of blocks used */ + PyObject *st_global; /* borrowed ref to st_top->ste_symbols */ + int st_nblocks; /* number of blocks used. kept for + consistency with the corresponding + compiler structure */ PyObject *st_private; /* name of current class or NULL */ - PyFutureFeatures *st_future; /* module's future features */ + PyFutureFeatures *st_future; /* module's future features that affect + the symbol table */ }; typedef struct _symtable_entry { @@ -34,7 +37,7 @@ typedef struct _symtable_entry { PyObject *ste_id; /* int: key in ste_table->st_blocks */ PyObject *ste_symbols; /* dict: variable names to flags */ PyObject *ste_name; /* string: name of current block */ - PyObject *ste_varnames; /* list of variable names */ + PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ _Py_block_ty ste_type; /* module, class, or function */ int ste_unoptimized; /* false if namespace is optimized */ diff --git a/Include/traceback.h b/Include/traceback.h index 69e3d05..7734707 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -5,6 +5,8 @@ extern "C" { #endif +#include "pystate.h" + struct _frame; /* Traceback interface */ @@ -28,6 +30,42 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int); PyAPI_DATA(PyTypeObject) PyTraceBack_Type; #define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type) +/* Write the Python traceback into the file 'fd'. For example: + + Traceback (most recent call first): + File "xxx", line xxx in <xxx> + File "xxx", line xxx in <xxx> + ... + File "xxx", line xxx in <xxx> + + This function is written for debug purpose only, to dump the traceback in + the worst case: after a segmentation fault, at fatal error, etc. That's why, + it is very limited. Strings are truncated to 100 characters and encoded to + ASCII with backslashreplace. It doesn't write the source code, only the + function name, filename and line number of each frame. Write only the first + 100 frames: if the traceback is truncated, write the line " ...". + + This function is signal safe. */ + +PyAPI_DATA(void) _Py_DumpTraceback( + int fd, + PyThreadState *tstate); + +/* Write the traceback of all threads into the file 'fd'. current_thread can be + NULL. Return NULL on success, or an error message on error. + + This function is written for debug purpose only. It calls + _Py_DumpTraceback() for each thread, and so has the same limitations. It + only write the traceback of the first 100 threads: write "..." if there are + more threads. + + This function is signal safe. */ + +PyAPI_DATA(const char*) _Py_DumpTracebackThreads( + int fd, PyInterpreterState *interp, + PyThreadState *current_thread); + + #ifdef __cplusplus } #endif diff --git a/Include/ucnhash.h b/Include/ucnhash.h index 70fdf13..8de9ba0 100644 --- a/Include/ucnhash.h +++ b/Include/ucnhash.h @@ -19,11 +19,13 @@ typedef struct { success, zero if not. Does not set Python exceptions. If self is NULL, data come from the default version of the database. If it is not NULL, it should be a unicodedata.ucd_X_Y_Z object */ - int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen); + int (*getname)(PyObject *self, Py_UCS4 code, char* buffer, int buflen, + int with_alias_and_seq); /* Get character code for a given name. Same error handling as for getname. */ - int (*getcode)(PyObject *self, const char* name, int namelen, Py_UCS4* code); + int (*getcode)(PyObject *self, const char* name, int namelen, Py_UCS4* code, + int with_named_seq); } _PyUnicode_Name_CAPI; diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 379a90c..c706fc2 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -64,16 +64,15 @@ Copyright (c) Corporation for National Research Initiatives. /* Python 3.x requires unicode */ #define Py_USING_UNICODE -/* FIXME: MvL's new implementation assumes that Py_UNICODE_SIZE is - properly set, but the default rules below doesn't set it. I'll - sort this out some other day -- fredrik@pythonware.com */ - -#ifndef Py_UNICODE_SIZE -#error Must define Py_UNICODE_SIZE +#ifndef SIZEOF_WCHAR_T +#error Must define SIZEOF_WCHAR_T #endif -/* Setting Py_UNICODE_WIDE enables UCS-4 storage. Otherwise, Unicode - strings are stored as UCS-2 (with limited support for UTF-16) */ +#define Py_UNICODE_SIZE SIZEOF_WCHAR_T + +/* If wchar_t can be used for UCS-4 storage, set Py_UNICODE_WIDE. + Otherwise, Unicode strings are stored as UCS-2 (with limited support + for UTF-16) */ #if Py_UNICODE_SIZE >= 4 #define Py_UNICODE_WIDE @@ -84,19 +83,14 @@ Copyright (c) Corporation for National Research Initiatives. /* #define HAVE_WCHAR_H */ /* #define HAVE_USABLE_WCHAR_T */ -/* Defaults for various platforms */ -#ifndef PY_UNICODE_TYPE - -/* Windows has a usable wchar_t type (unless we're using UCS-4) */ -# if defined(MS_WIN32) && Py_UNICODE_SIZE == 2 -# define HAVE_USABLE_WCHAR_T -# define PY_UNICODE_TYPE wchar_t -# endif - -# if defined(Py_UNICODE_WIDE) -# define PY_UNICODE_TYPE Py_UCS4 -# endif +/* Py_UNICODE was the native Unicode storage format (code unit) used by + Python and represents a single Unicode element in the Unicode type. + With PEP 393, Py_UNICODE is deprecated and replaced with a + typedef to wchar_t. */ +#ifndef Py_LIMITED_API +#define PY_UNICODE_TYPE wchar_t +typedef wchar_t Py_UNICODE; #endif /* If the compiler provides a wchar_t type we try to support it @@ -109,6 +103,10 @@ Copyright (c) Corporation for National Research Initiatives. # endif #endif +#if defined(MS_WINDOWS) +# define HAVE_MBCS +#endif + #ifdef HAVE_WCHAR_H /* Work around a cosmetic bug in BSDI 4.x wchar.h; thanks to Thomas Wouters */ # ifdef _HAVE_BSDI @@ -117,201 +115,24 @@ Copyright (c) Corporation for National Research Initiatives. # include <wchar.h> #endif -/* - * Use this typedef when you need to represent a UTF-16 surrogate pair - * as single unsigned integer. - */ -#if SIZEOF_INT >= 4 +/* Py_UCS4 and Py_UCS2 are typedefs for the respective + unicode representations. */ +#if SIZEOF_INT == 4 typedef unsigned int Py_UCS4; -#elif SIZEOF_LONG >= 4 +#elif SIZEOF_LONG == 4 typedef unsigned long Py_UCS4; +#else +#error "Could not find a proper typedef for Py_UCS4" #endif -/* Py_UNICODE is the native Unicode storage format (code unit) used by - Python and represents a single Unicode element in the Unicode - type. */ - -#ifndef Py_LIMITED_API -typedef PY_UNICODE_TYPE Py_UNICODE; -#endif - -/* --- UCS-2/UCS-4 Name Mangling ------------------------------------------ */ - -/* Unicode API names are mangled to assure that UCS-2 and UCS-4 builds - produce different external names and thus cause import errors in - case Python interpreters and extensions with mixed compiled in - Unicode width assumptions are combined. */ - -#ifndef Py_UNICODE_WIDE - -# define PyUnicode_AsASCIIString PyUnicodeUCS2_AsASCIIString -# define PyUnicode_AsCharmapString PyUnicodeUCS2_AsCharmapString -# define PyUnicode_AsDecodedObject PyUnicodeUCS2_AsDecodedObject -# define PyUnicode_AsDecodedUnicode PyUnicodeUCS2_AsDecodedUnicode -# define PyUnicode_AsEncodedObject PyUnicodeUCS2_AsEncodedObject -# define PyUnicode_AsEncodedString PyUnicodeUCS2_AsEncodedString -# define PyUnicode_AsEncodedUnicode PyUnicodeUCS2_AsEncodedUnicode -# define PyUnicode_AsLatin1String PyUnicodeUCS2_AsLatin1String -# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS2_AsRawUnicodeEscapeString -# define PyUnicode_AsUTF32String PyUnicodeUCS2_AsUTF32String -# define PyUnicode_AsUTF16String PyUnicodeUCS2_AsUTF16String -# define PyUnicode_AsUTF8String PyUnicodeUCS2_AsUTF8String -# define PyUnicode_AsUnicode PyUnicodeUCS2_AsUnicode -# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS2_AsUnicodeEscapeString -# define PyUnicode_AsWideChar PyUnicodeUCS2_AsWideChar -# define PyUnicode_AsWideCharString PyUnicodeUCS2_AsWideCharString -# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist -# define PyUnicode_Compare PyUnicodeUCS2_Compare -# define PyUnicode_CompareWithASCIIString PyUnicodeUCS2_CompareWithASCIIString -# define PyUnicode_Concat PyUnicodeUCS2_Concat -# define PyUnicode_Append PyUnicodeUCS2_Append -# define PyUnicode_AppendAndDel PyUnicodeUCS2_AppendAndDel -# define PyUnicode_Contains PyUnicodeUCS2_Contains -# define PyUnicode_Count PyUnicodeUCS2_Count -# define PyUnicode_Decode PyUnicodeUCS2_Decode -# define PyUnicode_DecodeASCII PyUnicodeUCS2_DecodeASCII -# define PyUnicode_DecodeCharmap PyUnicodeUCS2_DecodeCharmap -# define PyUnicode_DecodeLatin1 PyUnicodeUCS2_DecodeLatin1 -# define PyUnicode_DecodeFSDefault PyUnicodeUCS2_DecodeFSDefault -# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS2_DecodeFSDefaultAndSize -# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS2_DecodeRawUnicodeEscape -# define PyUnicode_DecodeUTF32 PyUnicodeUCS2_DecodeUTF32 -# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS2_DecodeUTF32Stateful -# define PyUnicode_DecodeUTF16 PyUnicodeUCS2_DecodeUTF16 -# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS2_DecodeUTF16Stateful -# define PyUnicode_DecodeUTF8 PyUnicodeUCS2_DecodeUTF8 -# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS2_DecodeUTF8Stateful -# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS2_DecodeUnicodeEscape -# define PyUnicode_Encode PyUnicodeUCS2_Encode -# define PyUnicode_EncodeASCII PyUnicodeUCS2_EncodeASCII -# define PyUnicode_EncodeCharmap PyUnicodeUCS2_EncodeCharmap -# define PyUnicode_EncodeDecimal PyUnicodeUCS2_EncodeDecimal -# define PyUnicode_EncodeLatin1 PyUnicodeUCS2_EncodeLatin1 -# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS2_EncodeRawUnicodeEscape -# define PyUnicode_EncodeUTF32 PyUnicodeUCS2_EncodeUTF32 -# define PyUnicode_EncodeUTF16 PyUnicodeUCS2_EncodeUTF16 -# define PyUnicode_EncodeUTF8 PyUnicodeUCS2_EncodeUTF8 -# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS2_EncodeUnicodeEscape -# define PyUnicode_Find PyUnicodeUCS2_Find -# define PyUnicode_Format PyUnicodeUCS2_Format -# define PyUnicode_FromEncodedObject PyUnicodeUCS2_FromEncodedObject -# define PyUnicode_FromFormat PyUnicodeUCS2_FromFormat -# define PyUnicode_FromFormatV PyUnicodeUCS2_FromFormatV -# define PyUnicode_FromObject PyUnicodeUCS2_FromObject -# define PyUnicode_FromOrdinal PyUnicodeUCS2_FromOrdinal -# define PyUnicode_FromString PyUnicodeUCS2_FromString -# define PyUnicode_FromStringAndSize PyUnicodeUCS2_FromStringAndSize -# define PyUnicode_FromUnicode PyUnicodeUCS2_FromUnicode -# define PyUnicode_FromWideChar PyUnicodeUCS2_FromWideChar -# define PyUnicode_FSConverter PyUnicodeUCS2_FSConverter -# define PyUnicode_FSDecoder PyUnicodeUCS2_FSDecoder -# define PyUnicode_GetDefaultEncoding PyUnicodeUCS2_GetDefaultEncoding -# define PyUnicode_GetMax PyUnicodeUCS2_GetMax -# define PyUnicode_GetSize PyUnicodeUCS2_GetSize -# define PyUnicode_IsIdentifier PyUnicodeUCS2_IsIdentifier -# define PyUnicode_Join PyUnicodeUCS2_Join -# define PyUnicode_Partition PyUnicodeUCS2_Partition -# define PyUnicode_RPartition PyUnicodeUCS2_RPartition -# define PyUnicode_RSplit PyUnicodeUCS2_RSplit -# define PyUnicode_Replace PyUnicodeUCS2_Replace -# define PyUnicode_Resize PyUnicodeUCS2_Resize -# define PyUnicode_RichCompare PyUnicodeUCS2_RichCompare -# define PyUnicode_Split PyUnicodeUCS2_Split -# define PyUnicode_Splitlines PyUnicodeUCS2_Splitlines -# define PyUnicode_Tailmatch PyUnicodeUCS2_Tailmatch -# define PyUnicode_Translate PyUnicodeUCS2_Translate -# define PyUnicode_TranslateCharmap PyUnicodeUCS2_TranslateCharmap -# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString -# define _PyUnicode_Fini _PyUnicodeUCS2_Fini -# define _PyUnicode_Init _PyUnicodeUCS2_Init -# define PyUnicode_strdup PyUnicodeUCS2_strdup - +#if SIZEOF_SHORT == 2 +typedef unsigned short Py_UCS2; #else - -# define PyUnicode_AsASCIIString PyUnicodeUCS4_AsASCIIString -# define PyUnicode_AsCharmapString PyUnicodeUCS4_AsCharmapString -# define PyUnicode_AsDecodedObject PyUnicodeUCS4_AsDecodedObject -# define PyUnicode_AsDecodedUnicode PyUnicodeUCS4_AsDecodedUnicode -# define PyUnicode_AsEncodedObject PyUnicodeUCS4_AsEncodedObject -# define PyUnicode_AsEncodedString PyUnicodeUCS4_AsEncodedString -# define PyUnicode_AsEncodedUnicode PyUnicodeUCS4_AsEncodedUnicode -# define PyUnicode_AsLatin1String PyUnicodeUCS4_AsLatin1String -# define PyUnicode_AsRawUnicodeEscapeString PyUnicodeUCS4_AsRawUnicodeEscapeString -# define PyUnicode_AsUTF32String PyUnicodeUCS4_AsUTF32String -# define PyUnicode_AsUTF16String PyUnicodeUCS4_AsUTF16String -# define PyUnicode_AsUTF8String PyUnicodeUCS4_AsUTF8String -# define PyUnicode_AsUnicode PyUnicodeUCS4_AsUnicode -# define PyUnicode_AsUnicodeEscapeString PyUnicodeUCS4_AsUnicodeEscapeString -# define PyUnicode_AsWideChar PyUnicodeUCS4_AsWideChar -# define PyUnicode_AsWideCharString PyUnicodeUCS4_AsWideCharString -# define PyUnicode_ClearFreeList PyUnicodeUCS4_ClearFreelist -# define PyUnicode_Compare PyUnicodeUCS4_Compare -# define PyUnicode_CompareWithASCIIString PyUnicodeUCS4_CompareWithASCIIString -# define PyUnicode_Concat PyUnicodeUCS4_Concat -# define PyUnicode_Append PyUnicodeUCS4_Append -# define PyUnicode_AppendAndDel PyUnicodeUCS4_AppendAndDel -# define PyUnicode_Contains PyUnicodeUCS4_Contains -# define PyUnicode_Count PyUnicodeUCS4_Count -# define PyUnicode_Decode PyUnicodeUCS4_Decode -# define PyUnicode_DecodeASCII PyUnicodeUCS4_DecodeASCII -# define PyUnicode_DecodeCharmap PyUnicodeUCS4_DecodeCharmap -# define PyUnicode_DecodeLatin1 PyUnicodeUCS4_DecodeLatin1 -# define PyUnicode_DecodeFSDefault PyUnicodeUCS4_DecodeFSDefault -# define PyUnicode_DecodeFSDefaultAndSize PyUnicodeUCS4_DecodeFSDefaultAndSize -# define PyUnicode_DecodeRawUnicodeEscape PyUnicodeUCS4_DecodeRawUnicodeEscape -# define PyUnicode_DecodeUTF32 PyUnicodeUCS4_DecodeUTF32 -# define PyUnicode_DecodeUTF32Stateful PyUnicodeUCS4_DecodeUTF32Stateful -# define PyUnicode_DecodeUTF16 PyUnicodeUCS4_DecodeUTF16 -# define PyUnicode_DecodeUTF16Stateful PyUnicodeUCS4_DecodeUTF16Stateful -# define PyUnicode_DecodeUTF8 PyUnicodeUCS4_DecodeUTF8 -# define PyUnicode_DecodeUTF8Stateful PyUnicodeUCS4_DecodeUTF8Stateful -# define PyUnicode_DecodeUnicodeEscape PyUnicodeUCS4_DecodeUnicodeEscape -# define PyUnicode_Encode PyUnicodeUCS4_Encode -# define PyUnicode_EncodeASCII PyUnicodeUCS4_EncodeASCII -# define PyUnicode_EncodeCharmap PyUnicodeUCS4_EncodeCharmap -# define PyUnicode_EncodeDecimal PyUnicodeUCS4_EncodeDecimal -# define PyUnicode_EncodeLatin1 PyUnicodeUCS4_EncodeLatin1 -# define PyUnicode_EncodeRawUnicodeEscape PyUnicodeUCS4_EncodeRawUnicodeEscape -# define PyUnicode_EncodeUTF32 PyUnicodeUCS4_EncodeUTF32 -# define PyUnicode_EncodeUTF16 PyUnicodeUCS4_EncodeUTF16 -# define PyUnicode_EncodeUTF8 PyUnicodeUCS4_EncodeUTF8 -# define PyUnicode_EncodeUnicodeEscape PyUnicodeUCS4_EncodeUnicodeEscape -# define PyUnicode_Find PyUnicodeUCS4_Find -# define PyUnicode_Format PyUnicodeUCS4_Format -# define PyUnicode_FromEncodedObject PyUnicodeUCS4_FromEncodedObject -# define PyUnicode_FromFormat PyUnicodeUCS4_FromFormat -# define PyUnicode_FromFormatV PyUnicodeUCS4_FromFormatV -# define PyUnicode_FromObject PyUnicodeUCS4_FromObject -# define PyUnicode_FromOrdinal PyUnicodeUCS4_FromOrdinal -# define PyUnicode_FromString PyUnicodeUCS4_FromString -# define PyUnicode_FromStringAndSize PyUnicodeUCS4_FromStringAndSize -# define PyUnicode_FromUnicode PyUnicodeUCS4_FromUnicode -# define PyUnicode_FromWideChar PyUnicodeUCS4_FromWideChar -# define PyUnicode_FSConverter PyUnicodeUCS4_FSConverter -# define PyUnicode_FSDecoder PyUnicodeUCS4_FSDecoder -# define PyUnicode_GetDefaultEncoding PyUnicodeUCS4_GetDefaultEncoding -# define PyUnicode_GetMax PyUnicodeUCS4_GetMax -# define PyUnicode_GetSize PyUnicodeUCS4_GetSize -# define PyUnicode_IsIdentifier PyUnicodeUCS4_IsIdentifier -# define PyUnicode_Join PyUnicodeUCS4_Join -# define PyUnicode_Partition PyUnicodeUCS4_Partition -# define PyUnicode_RPartition PyUnicodeUCS4_RPartition -# define PyUnicode_RSplit PyUnicodeUCS4_RSplit -# define PyUnicode_Replace PyUnicodeUCS4_Replace -# define PyUnicode_Resize PyUnicodeUCS4_Resize -# define PyUnicode_RichCompare PyUnicodeUCS4_RichCompare -# define PyUnicode_Split PyUnicodeUCS4_Split -# define PyUnicode_Splitlines PyUnicodeUCS4_Splitlines -# define PyUnicode_Tailmatch PyUnicodeUCS4_Tailmatch -# define PyUnicode_Translate PyUnicodeUCS4_Translate -# define PyUnicode_TranslateCharmap PyUnicodeUCS4_TranslateCharmap -# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString -# define _PyUnicode_Fini _PyUnicodeUCS4_Fini -# define _PyUnicode_Init _PyUnicodeUCS4_Init -# define PyUnicode_strdup PyUnicodeUCS4_strdup - +#error "Could not find a proper typedef for Py_UCS2" #endif +typedef unsigned char Py_UCS1; + /* --- Internal Unicode Operations ---------------------------------------- */ /* Since splitting on whitespace is an important use case, and @@ -350,7 +171,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE; Py_UNICODE_ISDIGIT(ch) || \ Py_UNICODE_ISNUMERIC(ch)) -#define Py_UNICODE_COPY(target, source, length) \ +#define Py_UNICODE_COPY(target, source, length) \ Py_MEMCPY((target), (source), (length)*sizeof(Py_UNICODE)) #define Py_UNICODE_FILL(target, value, length) \ @@ -358,13 +179,27 @@ typedef PY_UNICODE_TYPE Py_UNICODE; for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\ } while (0) +/* macros to work with surrogates */ +#define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDFFF) +#define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) +#define Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) +/* Join two surrogate characters and return a single Py_UCS4 value. */ +#define Py_UNICODE_JOIN_SURROGATES(high, low) \ + (((((Py_UCS4)(high) & 0x03FF) << 10) | \ + ((Py_UCS4)(low) & 0x03FF)) + 0x10000) +/* high surrogate = top 10 bits added to D800 */ +#define Py_UNICODE_HIGH_SURROGATE(ch) (0xD800 | (((ch) - 0x10000) >> 10)) +/* low surrogate = bottom 10 bits added to DC00 */ +#define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 | (((ch) - 0x10000) & 0x3FF)) + /* Check if substring matches at given offset. The offset must be valid, and the substring must not be empty. */ #define Py_UNICODE_MATCH(string, offset, substring) \ - ((*((string)->str + (offset)) == *((substring)->str)) && \ - ((*((string)->str + (offset) + (substring)->length-1) == *((substring)->str + (substring)->length-1))) && \ - !memcmp((string)->str + (offset), (substring)->str, (substring)->length*sizeof(Py_UNICODE))) + ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \ + ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \ + !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE))) + #endif /* Py_LIMITED_API */ #ifdef __cplusplus @@ -374,41 +209,374 @@ extern "C" { /* --- Unicode Type ------------------------------------------------------- */ #ifndef Py_LIMITED_API + +/* ASCII-only strings created through PyUnicode_New use the PyASCIIObject + structure. state.ascii and state.compact are set, and the data + immediately follow the structure. utf8_length and wstr_length can be found + in the length field; the utf8 pointer is equal to the data pointer. */ typedef struct { + /* There are 4 forms of Unicode strings: + + - compact ascii: + + * structure = PyASCIIObject + * test: PyUnicode_IS_COMPACT_ASCII(op) + * kind = PyUnicode_1BYTE_KIND + * compact = 1 + * ascii = 1 + * ready = 1 + * (length is the length of the utf8 and wstr strings) + * (data starts just after the structure) + * (since ASCII is decoded from UTF-8, the utf8 string are the data) + + - compact: + + * structure = PyCompactUnicodeObject + * test: PyUnicode_IS_COMPACT(op) && !PyUnicode_IS_ASCII(op) + * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or + PyUnicode_4BYTE_KIND + * compact = 1 + * ready = 1 + * ascii = 0 + * utf8 is not shared with data + * utf8_length = 0 if utf8 is NULL + * wstr is shared with data and wstr_length=length + if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2 + or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_t)=4 + * wstr_length = 0 if wstr is NULL + * (data starts just after the structure) + + - legacy string, not ready: + + * structure = PyUnicodeObject + * test: kind == PyUnicode_WCHAR_KIND + * length = 0 (use wstr_length) + * hash = -1 + * kind = PyUnicode_WCHAR_KIND + * compact = 0 + * ascii = 0 + * ready = 0 + * interned = SSTATE_NOT_INTERNED + * wstr is not NULL + * data.any is NULL + * utf8 is NULL + * utf8_length = 0 + + - legacy string, ready: + + * structure = PyUnicodeObject structure + * test: !PyUnicode_IS_COMPACT(op) && kind != PyUnicode_WCHAR_KIND + * kind = PyUnicode_1BYTE_KIND, PyUnicode_2BYTE_KIND or + PyUnicode_4BYTE_KIND + * compact = 0 + * ready = 1 + * data.any is not NULL + * utf8 is shared and utf8_length = length with data.any if ascii = 1 + * utf8_length = 0 if utf8 is NULL + * wstr is shared with data.any and wstr_length = length + if kind=PyUnicode_2BYTE_KIND and sizeof(wchar_t)=2 + or if kind=PyUnicode_4BYTE_KIND and sizeof(wchar_4)=4 + * wstr_length = 0 if wstr is NULL + + Compact strings use only one memory block (structure + characters), + whereas legacy strings use one block for the structure and one block + for characters. + + Legacy strings are created by PyUnicode_FromUnicode() and + PyUnicode_FromStringAndSize(NULL, size) functions. They become ready + when PyUnicode_READY() is called. + + See also _PyUnicode_CheckConsistency(). + */ PyObject_HEAD - Py_ssize_t length; /* Length of raw Unicode data in buffer */ - Py_UNICODE *str; /* Raw Unicode buffer */ + Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ - int state; /* != 0 if interned. In this case the two - * references from the dictionary to this object - * are *not* counted in ob_refcnt. */ - PyObject *defenc; /* (Default) Encoded version as Python - string, or NULL; this is used for - implementing the buffer protocol */ + struct { + /* + SSTATE_NOT_INTERNED (0) + SSTATE_INTERNED_MORTAL (1) + SSTATE_INTERNED_IMMORTAL (2) + + If interned != SSTATE_NOT_INTERNED, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + */ + unsigned int interned:2; + /* Character size: + + - PyUnicode_WCHAR_KIND (0): + + * character type = wchar_t (16 or 32 bits, depending on the + platform) + + - PyUnicode_1BYTE_KIND (1): + + * character type = Py_UCS1 (8 bits, unsigned) + * all characters are in the range U+0000-U+00FF (latin1) + * if ascii is set, all characters are in the range U+0000-U+007F + (ASCII), otherwise at least one character is in the range + U+0080-U+00FF + + - PyUnicode_2BYTE_KIND (2): + + * character type = Py_UCS2 (16 bits, unsigned) + * all characters are in the range U+0000-U+FFFF (BMP) + * at least one character is in the range U+0100-U+FFFF + + - PyUnicode_4BYTE_KIND (4): + + * character type = Py_UCS4 (32 bits, unsigned) + * all characters are in the range U+0000-U+10FFFF + * at least one character is in the range U+10000-U+10FFFF + */ + unsigned int kind:3; + /* Compact is with respect to the allocation scheme. Compact unicode + objects only require one memory block while non-compact objects use + one block for the PyUnicodeObject struct and another for its data + buffer. */ + unsigned int compact:1; + /* The string only contains characters in the range U+0000-U+007F (ASCII) + and the kind is PyUnicode_1BYTE_KIND. If ascii is set and compact is + set, use the PyASCIIObject structure. */ + unsigned int ascii:1; + /* The ready flag indicates whether the object layout is initialized + completely. This means that this is either a compact object, or + the data pointer is filled out. The bit is redundant, and helps + to minimize the test in PyUnicode_IS_READY(). */ + unsigned int ready:1; + } state; + wchar_t *wstr; /* wchar_t representation (null-terminated) */ +} PyASCIIObject; + +/* Non-ASCII strings allocated through PyUnicode_New use the + PyCompactUnicodeObject structure. state.compact is set, and the data + immediately follow the structure. */ +typedef struct { + PyASCIIObject _base; + Py_ssize_t utf8_length; /* Number of bytes in utf8, excluding the + * terminating \0. */ + char *utf8; /* UTF-8 representation (null-terminated) */ + Py_ssize_t wstr_length; /* Number of code points in wstr, possible + * surrogates count as two code points. */ +} PyCompactUnicodeObject; + +/* Strings allocated through PyUnicode_FromUnicode(NULL, len) use the + PyUnicodeObject structure. The actual string data is initially in the wstr + block, and copied into the data block using _PyUnicode_Ready. */ +typedef struct { + PyCompactUnicodeObject _base; + union { + void *any; + Py_UCS1 *latin1; + Py_UCS2 *ucs2; + Py_UCS4 *ucs4; + } data; /* Canonical, smallest-form Unicode buffer */ } PyUnicodeObject; #endif PyAPI_DATA(PyTypeObject) PyUnicode_Type; PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type; -#define SSTATE_NOT_INTERNED 0 -#define SSTATE_INTERNED_MORTAL 1 -#define SSTATE_INTERNED_IMMORTAL 2 - #define PyUnicode_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type) /* Fast access macros */ #ifndef Py_LIMITED_API -#define PyUnicode_GET_SIZE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length)) + +#define PyUnicode_WSTR_LENGTH(op) \ + (PyUnicode_IS_COMPACT_ASCII(op) ? \ + ((PyASCIIObject*)op)->length : \ + ((PyCompactUnicodeObject*)op)->wstr_length) + +/* Returns the deprecated Py_UNICODE representation's size in code units + (this includes surrogate pairs as 2 units). + If the Py_UNICODE representation is not available, it will be computed + on request. Use PyUnicode_GET_LENGTH() for the length in code points. */ + +#define PyUnicode_GET_SIZE(op) \ + (assert(PyUnicode_Check(op)), \ + (((PyASCIIObject *)(op))->wstr) ? \ + PyUnicode_WSTR_LENGTH(op) : \ + ((void)PyUnicode_AsUnicode((PyObject *)(op)), \ + assert(((PyASCIIObject *)(op))->wstr), \ + PyUnicode_WSTR_LENGTH(op))) + #define PyUnicode_GET_DATA_SIZE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))) + (PyUnicode_GET_SIZE(op) * Py_UNICODE_SIZE) + +/* Alias for PyUnicode_AsUnicode(). This will create a wchar_t/Py_UNICODE + representation on demand. Using this macro is very inefficient now, + try to port your code to use the new PyUnicode_*BYTE_DATA() macros or + use PyUnicode_WRITE() and PyUnicode_READ(). */ + #define PyUnicode_AS_UNICODE(op) \ - (assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->str)) + (assert(PyUnicode_Check(op)), \ + (((PyASCIIObject *)(op))->wstr) ? (((PyASCIIObject *)(op))->wstr) : \ + PyUnicode_AsUnicode((PyObject *)(op))) + #define PyUnicode_AS_DATA(op) \ - (assert(PyUnicode_Check(op)),((const char *)((PyUnicodeObject *)(op))->str)) + ((const char *)(PyUnicode_AS_UNICODE(op))) + + +/* --- Flexible String Representation Helper Macros (PEP 393) -------------- */ + +/* Values for PyASCIIObject.state: */ + +/* Interning state. */ +#define SSTATE_NOT_INTERNED 0 +#define SSTATE_INTERNED_MORTAL 1 +#define SSTATE_INTERNED_IMMORTAL 2 + +/* Return true if the string contains only ASCII characters, or 0 if not. The + string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be + ready. */ +#define PyUnicode_IS_ASCII(op) \ + (assert(PyUnicode_Check(op)), \ + assert(PyUnicode_IS_READY(op)), \ + ((PyASCIIObject*)op)->state.ascii) + +/* Return true if the string is compact or 0 if not. + No type checks or Ready calls are performed. */ +#define PyUnicode_IS_COMPACT(op) \ + (((PyASCIIObject*)(op))->state.compact) + +/* Return true if the string is a compact ASCII string (use PyASCIIObject + structure), or 0 if not. No type checks or Ready calls are performed. */ +#define PyUnicode_IS_COMPACT_ASCII(op) \ + (((PyASCIIObject*)op)->state.ascii && PyUnicode_IS_COMPACT(op)) + +enum PyUnicode_Kind { +/* String contains only wstr byte characters. This is only possible + when the string was created with a legacy API and _PyUnicode_Ready() + has not been called yet. */ + PyUnicode_WCHAR_KIND = 0, +/* Return values of the PyUnicode_KIND() macro: */ + PyUnicode_1BYTE_KIND = 1, + PyUnicode_2BYTE_KIND = 2, + PyUnicode_4BYTE_KIND = 4 +}; + +/* Return pointers to the canonical representation cast to unsigned char, + Py_UCS2, or Py_UCS4 for direct character access. + No checks are performed, use PyUnicode_KIND() before to ensure + these will work correctly. */ + +#define PyUnicode_1BYTE_DATA(op) ((Py_UCS1*)PyUnicode_DATA(op)) +#define PyUnicode_2BYTE_DATA(op) ((Py_UCS2*)PyUnicode_DATA(op)) +#define PyUnicode_4BYTE_DATA(op) ((Py_UCS4*)PyUnicode_DATA(op)) + +/* Return one of the PyUnicode_*_KIND values defined above. */ +#define PyUnicode_KIND(op) \ + (assert(PyUnicode_Check(op)), \ + assert(PyUnicode_IS_READY(op)), \ + ((PyASCIIObject *)(op))->state.kind) + +/* Return a void pointer to the raw unicode buffer. */ +#define _PyUnicode_COMPACT_DATA(op) \ + (PyUnicode_IS_ASCII(op) ? \ + ((void*)((PyASCIIObject*)(op) + 1)) : \ + ((void*)((PyCompactUnicodeObject*)(op) + 1))) + +#define _PyUnicode_NONCOMPACT_DATA(op) \ + (assert(((PyUnicodeObject*)(op))->data.any), \ + ((((PyUnicodeObject *)(op))->data.any))) + +#define PyUnicode_DATA(op) \ + (assert(PyUnicode_Check(op)), \ + PyUnicode_IS_COMPACT(op) ? _PyUnicode_COMPACT_DATA(op) : \ + _PyUnicode_NONCOMPACT_DATA(op)) + +/* In the access macros below, "kind" may be evaluated more than once. + All other macro parameters are evaluated exactly once, so it is safe + to put side effects into them (such as increasing the index). */ + +/* Write into the canonical representation, this macro does not do any sanity + checks and is intended for usage in loops. The caller should cache the + kind and data pointers obtained from other macro calls. + index is the index in the string (starts at 0) and value is the new + code point value which should be written to that location. */ +#define PyUnicode_WRITE(kind, data, index, value) \ + do { \ + switch ((kind)) { \ + case PyUnicode_1BYTE_KIND: { \ + ((Py_UCS1 *)(data))[(index)] = (Py_UCS1)(value); \ + break; \ + } \ + case PyUnicode_2BYTE_KIND: { \ + ((Py_UCS2 *)(data))[(index)] = (Py_UCS2)(value); \ + break; \ + } \ + default: { \ + assert((kind) == PyUnicode_4BYTE_KIND); \ + ((Py_UCS4 *)(data))[(index)] = (Py_UCS4)(value); \ + } \ + } \ + } while (0) + +/* Read a code point from the string's canonical representation. No checks + or ready calls are performed. */ +#define PyUnicode_READ(kind, data, index) \ + ((Py_UCS4) \ + ((kind) == PyUnicode_1BYTE_KIND ? \ + ((const Py_UCS1 *)(data))[(index)] : \ + ((kind) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(data))[(index)] : \ + ((const Py_UCS4 *)(data))[(index)] \ + ) \ + )) + +/* PyUnicode_READ_CHAR() is less efficient than PyUnicode_READ() because it + calls PyUnicode_KIND() and might call it twice. For single reads, use + PyUnicode_READ_CHAR, for multiple consecutive reads callers should + cache kind and use PyUnicode_READ instead. */ +#define PyUnicode_READ_CHAR(unicode, index) \ + (assert(PyUnicode_Check(unicode)), \ + assert(PyUnicode_IS_READY(unicode)), \ + (Py_UCS4) \ + (PyUnicode_KIND((unicode)) == PyUnicode_1BYTE_KIND ? \ + ((const Py_UCS1 *)(PyUnicode_DATA((unicode))))[(index)] : \ + (PyUnicode_KIND((unicode)) == PyUnicode_2BYTE_KIND ? \ + ((const Py_UCS2 *)(PyUnicode_DATA((unicode))))[(index)] : \ + ((const Py_UCS4 *)(PyUnicode_DATA((unicode))))[(index)] \ + ) \ + )) + +/* Returns the length of the unicode string. The caller has to make sure that + the string has it's canonical representation set before calling + this macro. Call PyUnicode_(FAST_)Ready to ensure that. */ +#define PyUnicode_GET_LENGTH(op) \ + (assert(PyUnicode_Check(op)), \ + assert(PyUnicode_IS_READY(op)), \ + ((PyASCIIObject *)(op))->length) + + +/* Fast check to determine whether an object is ready. Equivalent to + PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */ + +#define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready) + +/* PyUnicode_READY() does less work than _PyUnicode_Ready() in the best + case. If the canonical representation is not yet set, it will still call + _PyUnicode_Ready(). + Returns 0 on success and -1 on errors. */ +#define PyUnicode_READY(op) \ + (assert(PyUnicode_Check(op)), \ + (PyUnicode_IS_READY(op) ? \ + 0 : _PyUnicode_Ready((PyObject *)(op)))) + +/* Return a maximum character value which is suitable for creating another + string based on op. This is always an approximation but more efficient + than iterating over the string. */ +#define PyUnicode_MAX_CHAR_VALUE(op) \ + (assert(PyUnicode_IS_READY(op)), \ + (PyUnicode_IS_ASCII(op) ? \ + (0x7f) : \ + (PyUnicode_KIND(op) == PyUnicode_1BYTE_KIND ? \ + (0xffU) : \ + (PyUnicode_KIND(op) == PyUnicode_2BYTE_KIND ? \ + (0xffffU) : \ + (0x10ffffU))))) + #endif /* --- Constants ---------------------------------------------------------- */ @@ -418,12 +586,87 @@ PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type; Unicode character U+FFFD is the official REPLACEMENT CHARACTER in Unicode 3.0. */ -#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UNICODE) 0xFFFD) +#define Py_UNICODE_REPLACEMENT_CHARACTER ((Py_UCS4) 0xFFFD) /* === Public API ========================================================= */ /* --- Plain Py_UNICODE --------------------------------------------------- */ +/* With PEP 393, this is the recommended way to allocate a new unicode object. + This function will allocate the object and its buffer in a single memory + block. Objects created using this function are not resizable. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) PyUnicode_New( + Py_ssize_t size, /* Number of code points in the new string */ + Py_UCS4 maxchar /* maximum code point value in the string */ + ); +#endif + +/* Initializes the canonical string representation from a the deprecated + wstr/Py_UNICODE representation. This function is used to convert Unicode + objects which were created using the old API to the new flexible format + introduced with PEP 393. + + Don't call this function directly, use the public PyUnicode_READY() macro + instead. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(int) _PyUnicode_Ready( + PyObject *unicode /* Unicode object */ + ); +#endif + +/* Get a copy of a Unicode string. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_Copy( + PyObject *unicode + ); +#endif + +/* Copy character from one unicode object into another, this function performs + character conversion when necessary and falls back to memcpy() if possible. + + Fail if to is too small (smaller than *how_many* or smaller than + len(from)-from_start), or if kind(from[from_start:from_start+how_many]) > + kind(to), or if *to* has more than 1 reference. + + Return the number of written character, or return -1 and raise an exception + on error. + + Pseudo-code: + + how_many = min(how_many, len(from) - from_start) + to[to_start:to_start+how_many] = from[from_start:from_start+how_many] + return how_many + + Note: The function doesn't write a terminating null character. + */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_ssize_t) PyUnicode_CopyCharacters( + PyObject *to, + Py_ssize_t to_start, + PyObject *from, + Py_ssize_t from_start, + Py_ssize_t how_many + ); +#endif + +/* Fill a string with a character: write fill_char into + unicode[start:start+length]. + + Fail if fill_char is bigger than the string maximum character, or if the + string has more than 1 reference. + + Return the number of written character, or return -1 and raise an exception + on error. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_ssize_t) PyUnicode_Fill( + PyObject *unicode, + Py_ssize_t start, + Py_ssize_t length, + Py_UCS4 fill_char + ); +#endif + /* Create a Unicode Object from the Py_UNICODE buffer u of the given size. @@ -448,13 +691,45 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromStringAndSize( ); /* Similar to PyUnicode_FromUnicode(), but u points to null-terminated - UTF-8 encoded bytes */ + UTF-8 encoded bytes. The size is determined with strlen(). */ PyAPI_FUNC(PyObject*) PyUnicode_FromString( const char *u /* UTF-8 encoded string */ ); +/* Create a new string from a buffer of Py_UCS1, Py_UCS2 or Py_UCS4 characters. + Scan the string to find the maximum character. */ +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( + int kind, + const void *buffer, + Py_ssize_t size); +#endif + +PyAPI_FUNC(PyObject*) PyUnicode_Substring( + PyObject *str, + Py_ssize_t start, + Py_ssize_t end); + +/* Copy the string into a UCS4 buffer including the null character if copy_null + is set. Return NULL and raise an exception on error. Raise a ValueError if + the buffer is smaller than the string. Return buffer on success. + + buflen is the length of the buffer in (Py_UCS4) characters. */ +PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4( + PyObject *unicode, + Py_UCS4* buffer, + Py_ssize_t buflen, + int copy_null); + +/* Copy the string into a UCS4 buffer. A new buffer is allocated using + * PyMem_Malloc; if this fails, NULL is returned with a memory error + exception set. */ +PyAPI_FUNC(Py_UCS4*) PyUnicode_AsUCS4Copy(PyObject *unicode); + /* Return a read-only pointer to the Unicode object's internal - Py_UNICODE buffer. */ + Py_UNICODE buffer. + If the wchar_t/Py_UNICODE representation is not yet available, this + function will calculate it. */ #ifndef Py_LIMITED_API PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( @@ -462,30 +737,69 @@ PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( ); #endif +/* Return a read-only pointer to the Unicode object's internal + Py_UNICODE buffer and save the length at size. + If the wchar_t/Py_UNICODE representation is not yet available, this + function will calculate it. */ + +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( + PyObject *unicode, /* Unicode object */ + Py_ssize_t *size /* location where to save the length */ + ); +#endif + /* Get the length of the Unicode object. */ +PyAPI_FUNC(Py_ssize_t) PyUnicode_GetLength( + PyObject *unicode +); + +/* Get the number of Py_UNICODE units in the + string representation. */ + PyAPI_FUNC(Py_ssize_t) PyUnicode_GetSize( PyObject *unicode /* Unicode object */ ); +/* Read a character from the string. */ + +PyAPI_FUNC(Py_UCS4) PyUnicode_ReadChar( + PyObject *unicode, + Py_ssize_t index + ); + +/* Write a character to the string. The string must have been created through + PyUnicode_New, must not be shared, and must not have been hashed yet. + + Return 0 on success, -1 on error. */ + +PyAPI_FUNC(int) PyUnicode_WriteChar( + PyObject *unicode, + Py_ssize_t index, + Py_UCS4 character + ); + #ifndef Py_LIMITED_API /* Get the maximum ordinal for a Unicode character. */ PyAPI_FUNC(Py_UNICODE) PyUnicode_GetMax(void); #endif -/* Resize an already allocated Unicode object to the new size length. +/* Resize an Unicode object. The length is the number of characters, except + if the kind of the string is PyUnicode_WCHAR_KIND: in this case, the length + is the number of Py_UNICODE characters. *unicode is modified to point to the new (resized) object and 0 returned on success. - This API may only be called by the function which also called the - Unicode constructor. The refcount on the object must be 1. Otherwise, - an error is returned. + Try to resize the string in place (which is usually faster than allocating + a new string and copy characters), or create a new string. Error handling is implemented as follows: an exception is set, -1 is returned and *unicode left untouched. -*/ + WARNING: The function doesn't check string content, the result may not be a + string in canonical representation. */ PyAPI_FUNC(int) PyUnicode_Resize( PyObject **unicode, /* Pointer to the Unicode object */ @@ -545,8 +859,9 @@ PyAPI_FUNC(PyObject *) PyUnicode_FromFormat( /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyUnicode_FormatAdvanced(PyObject *obj, - Py_UNICODE *format_spec, - Py_ssize_t format_spec_len); + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); #endif PyAPI_FUNC(void) PyUnicode_InternInPlace(PyObject **); @@ -559,7 +874,8 @@ PyAPI_FUNC(void) _Py_ReleaseInternedUnicodeStrings(void); #endif /* Use only if you know it's a string */ -#define PyUnicode_CHECK_INTERNED(op) (((PyUnicodeObject *)(op))->state) +#define PyUnicode_CHECK_INTERNED(op) \ + (((PyASCIIObject *)(op))->state.interned) /* --- wchar_t support for platforms which support it --------------------- */ @@ -606,6 +922,10 @@ PyAPI_FUNC(wchar_t*) PyUnicode_AsWideCharString( Py_ssize_t *size /* number of characters of the result */ ); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void*) _PyUnicode_AsKind(PyObject *s, unsigned int kind); +#endif + #endif /* --- Unicode ordinals --------------------------------------------------- */ @@ -651,50 +971,42 @@ PyAPI_FUNC(int) PyUnicode_ClearFreeList(void); /* --- Manage the default encoding ---------------------------------------- */ -/* Return a Python string holding the default encoded value of the - Unicode object. - - Same as PyUnicode_AsUTF8String() except - the resulting string is cached in the Unicode object for subsequent - usage by this function. The cached version is needed to implement - the character buffer interface and will live (at least) as long as - the Unicode object itself. - - The refcount of the string is *not* incremented. - - *** Exported for internal use by the interpreter only !!! *** - -*/ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString( - PyObject *unicode, - const char *errors); -#endif - /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode and the size of the encoded representation in bytes stored in *size. In case of an error, no *size is set. + This function caches the UTF-8 encoded string in the unicodeobject + and subsequent calls will return the same string. The memory is released + when the unicodeobject is deallocated. + + _PyUnicode_AsStringAndSize is a #define for PyUnicode_AsUTF8AndSize to + support the previous internal function with the same behaviour. + *** This API is for interpreter INTERNAL USE ONLY and will likely *** be removed or changed in the future. *** If you need to access the Unicode object as UTF-8 bytes string, *** please use PyUnicode_AsUTF8String() instead. - */ #ifndef Py_LIMITED_API -PyAPI_FUNC(char *) _PyUnicode_AsStringAndSize( +PyAPI_FUNC(char *) PyUnicode_AsUTF8AndSize( PyObject *unicode, Py_ssize_t *size); +#define _PyUnicode_AsStringAndSize PyUnicode_AsUTF8AndSize #endif /* Returns a pointer to the default encoding (UTF-8) of the Unicode object unicode. + Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation + in the unicodeobject. + + _PyUnicode_AsString is a #define for PyUnicode_AsUTF8 to + support the previous internal function with the same behaviour. + Use of this API is DEPRECATED since no size information can be extracted from the returned data. @@ -707,7 +1019,8 @@ PyAPI_FUNC(char *) _PyUnicode_AsStringAndSize( */ #ifndef Py_LIMITED_API -PyAPI_FUNC(char *) _PyUnicode_AsString(PyObject *unicode); +PyAPI_FUNC(char *) PyUnicode_AsUTF8(PyObject *unicode); +#define _PyUnicode_AsString PyUnicode_AsUTF8 #endif /* Returns "utf-8". */ @@ -812,6 +1125,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF7( int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ const char *errors /* error handling */ ); +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF7( + PyObject *unicode, /* Unicode object */ + int base64SetO, /* Encode RFC2152 Set O characters in base64 */ + int base64WhiteSpace, /* Encode whitespace (sp, ht, nl, cr) in base64 */ + const char *errors /* error handling */ + ); #endif /* --- UTF-8 Codecs ------------------------------------------------------- */ @@ -834,6 +1153,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String( ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String( + PyObject *unicode, + const char *errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF8( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* number of Py_UNICODE chars to encode */ @@ -915,6 +1238,11 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF32( const char *errors, /* error handling */ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ ); +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32( + PyObject *object, /* Unicode object */ + const char *errors, /* error handling */ + int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ + ); #endif /* --- UTF-16 Codecs ------------------------------------------------------ */ @@ -995,6 +1323,11 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeUTF16( const char *errors, /* error handling */ int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ ); +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( + PyObject* unicode, /* Unicode object */ + const char *errors, /* error handling */ + int byteorder /* byteorder to use 0=BOM+native;-1=LE,1=BE */ + ); #endif /* --- Unicode-Escape Codecs ---------------------------------------------- */ @@ -1064,6 +1397,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsLatin1String( ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( + PyObject* unicode, + const char* errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeLatin1( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ @@ -1088,6 +1425,10 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsASCIIString( ); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_AsASCIIString( + PyObject* unicode, + const char* errors); + PyAPI_FUNC(PyObject*) PyUnicode_EncodeASCII( const Py_UNICODE *data, /* Unicode char buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ @@ -1139,6 +1480,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCharmap( (unicode ordinal -> char ordinal) */ const char *errors /* error handling */ ); +PyAPI_FUNC(PyObject*) _PyUnicode_EncodeCharmap( + PyObject *unicode, /* Unicode object */ + PyObject *mapping, /* character mapping + (unicode ordinal -> char ordinal) */ + const char *errors /* error handling */ + ); #endif /* Translate a Py_UNICODE buffer of the given length by applying a @@ -1163,7 +1510,7 @@ PyAPI_FUNC(PyObject *) PyUnicode_TranslateCharmap( ); #endif -#ifdef MS_WIN32 +#ifdef HAVE_MBCS /* --- MBCS codecs for Windows -------------------------------------------- */ @@ -1180,6 +1527,14 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful( Py_ssize_t *consumed /* bytes consumed */ ); +PyAPI_FUNC(PyObject*) PyUnicode_DecodeCodePageStateful( + int code_page, /* code page number */ + const char *string, /* encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ + ); + PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString( PyObject *unicode /* Unicode object */ ); @@ -1187,12 +1542,18 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString( #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS( const Py_UNICODE *data, /* Unicode char buffer */ - Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ + Py_ssize_t length, /* number of Py_UNICODE chars to encode */ const char *errors /* error handling */ ); #endif -#endif /* MS_WIN32 */ +PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage( + int code_page, /* code page number */ + PyObject *unicode, /* Unicode object */ + const char *errors /* error handling */ + ); + +#endif /* HAVE_MBCS */ /* --- Decimal Encoder ---------------------------------------------------- */ @@ -1240,6 +1601,49 @@ PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII( ); #endif +/* Similar to PyUnicode_TransformDecimalToASCII(), but takes a PyObject + as argument instead of a raw buffer and length. This function additionally + transforms spaces to ASCII because this is what the callers in longobject, + floatobject, and complexobject did anyways. */ + +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( + PyObject *unicode /* Unicode object */ + ); +#endif + +/* --- Locale encoding --------------------------------------------------- */ + +/* Decode a string from the current locale encoding. The decoder is strict if + *surrogateescape* is equal to zero, otherwise it uses the 'surrogateescape' + error handler (PEP 383) to escape undecodable bytes. If a byte sequence can + be decoded as a surrogate character and *surrogateescape* is not equal to + zero, the byte sequence is escaped using the 'surrogateescape' error handler + instead of being decoded. *str* must end with a null character but cannot + contain embedded null characters. */ + +PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocaleAndSize( + const char *str, + Py_ssize_t len, + const char *errors); + +/* Similar to PyUnicode_DecodeLocaleAndSize(), but compute the string + length using strlen(). */ + +PyAPI_FUNC(PyObject*) PyUnicode_DecodeLocale( + const char *str, + const char *errors); + +/* Encode a Unicode object to the current locale encoding. The encoder is + strict is *surrogateescape* is equal to zero, otherwise the + "surrogateescape" error handler is used. Return a bytes object. The string + cannot contain embedded null characters.. */ + +PyAPI_FUNC(PyObject*) PyUnicode_EncodeLocale( + PyObject *unicode, + const char *errors + ); + /* --- File system encoding ---------------------------------------------- */ /* ParseTuple converter: encode str objects to bytes using @@ -1292,7 +1696,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeFSDefault( These are capable of handling Unicode objects and strings on input (we refer to them as strings in the descriptions) and return - Unicode objects or integers as apporpriate. */ + Unicode objects or integers as appropriate. */ /* Concat two strings giving a new Unicode string. */ @@ -1427,6 +1831,15 @@ PyAPI_FUNC(Py_ssize_t) PyUnicode_Find( int direction /* Find direction: +1 forward, -1 backward */ ); +/* Like PyUnicode_Find, but search for single character only. */ +PyAPI_FUNC(Py_ssize_t) PyUnicode_FindChar( + PyObject *str, + Py_UCS4 ch, + Py_ssize_t start, + Py_ssize_t end, + int direction + ); + /* Count the number of occurrences of substr in str[start:end]. */ PyAPI_FUNC(Py_ssize_t) PyUnicode_Count( @@ -1463,7 +1876,7 @@ PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString( /* Rich compare two strings and return one of the following: - NULL in case an exception was raised - - Py_True or Py_False for successfuly comparisons + - Py_True or Py_False for successfully comparisons - Py_NotImplemented in case the type combination is unknown Note that Py_EQ and Py_NE comparisons can cause a UnicodeWarning in @@ -1514,7 +1927,7 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s); #ifndef Py_LIMITED_API /* Externally visible for str.strip(unicode) */ PyAPI_FUNC(PyObject *) _PyUnicode_XStrip( - PyUnicodeObject *self, + PyObject *self, int striptype, PyObject *sepobj ); @@ -1536,13 +1949,16 @@ PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGroupingLocale(Py_UNICODE *buff into the string pointed to by buffer. For the argument descriptions, see Objects/stringlib/localeutil.h */ #ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(Py_UNICODE *buffer, - Py_ssize_t n_buffer, - Py_UNICODE *digits, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - const char *thousands_sep); +PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping( + PyObject *unicode, + int kind, + void *buffer, + Py_ssize_t n_buffer, + void *digits, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + const char *thousands_sep); #endif /* === Characters Type APIs =============================================== */ @@ -1598,6 +2014,34 @@ PyAPI_FUNC(Py_UCS4) _PyUnicode_ToTitlecase( Py_UCS4 ch /* Unicode character */ ); +PyAPI_FUNC(int) _PyUnicode_ToLowerFull( + Py_UCS4 ch, /* Unicode character */ + Py_UCS4 *res + ); + +PyAPI_FUNC(int) _PyUnicode_ToTitleFull( + Py_UCS4 ch, /* Unicode character */ + Py_UCS4 *res + ); + +PyAPI_FUNC(int) _PyUnicode_ToUpperFull( + Py_UCS4 ch, /* Unicode character */ + Py_UCS4 *res + ); + +PyAPI_FUNC(int) _PyUnicode_ToFoldedFull( + Py_UCS4 ch, /* Unicode character */ + Py_UCS4 *res + ); + +PyAPI_FUNC(int) _PyUnicode_IsCaseIgnorable( + Py_UCS4 ch /* Unicode character */ + ); + +PyAPI_FUNC(int) _PyUnicode_IsCased( + Py_UCS4 ch /* Unicode character */ + ); + PyAPI_FUNC(int) _PyUnicode_ToDecimalDigit( Py_UCS4 ch /* Unicode character */ ); @@ -1676,6 +2120,17 @@ PyAPI_FUNC(Py_UNICODE*) PyUnicode_AsUnicodeCopy( ); #endif /* Py_LIMITED_API */ +#if defined(Py_DEBUG) && !defined(Py_LIMITED_API) +PyAPI_FUNC(int) _PyUnicode_CheckConsistency( + PyObject *op, + int check_content); +#endif + +/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/ +PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); +/* Clear all static strings. */ +PyAPI_FUNC(void) _PyUnicode_ClearStaticStrings(void); + #ifdef __cplusplus } #endif |