Add ast.Constant

Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by
external AST optimizers, but the compiler does not emit directly such node.

An optimizer can replace the following AST nodes with ast.Constant:

* ast.NameConstant: None, False, True
* ast.Num: int, float, complex
* ast.Str: str
* ast.Bytes: bytes
* ast.Tuple if items are constants too: tuple
* frozenset

Update code to accept ast.Constant instead of ast.Num and/or ast.Str:

* compiler
* docstrings
* ast.literal_eval()
* Tools/parser/unparse.py

5 files changed, 181 insertions, 14 deletions

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 07d9b3e..4dde11f 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -306,6 +306,10 @@ static char *NameConstant_fields[]={
     "value",
 };
 static PyTypeObject *Ellipsis_type;
+static PyTypeObject *Constant_type;
+static char *Constant_fields[]={
+    "value",
+};
 static PyTypeObject *Attribute_type;
 _Py_IDENTIFIER(attr);
 _Py_IDENTIFIER(ctx);
@@ -709,6 +713,7 @@ static PyObject* ast2obj_object(void *o)
     return (PyObject*)o;
 }
 #define ast2obj_singleton ast2obj_object
+#define ast2obj_constant ast2obj_object
 #define ast2obj_identifier ast2obj_object
 #define ast2obj_string ast2obj_object
 #define ast2obj_bytes ast2obj_object
@@ -746,6 +751,26 @@ static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
     return 0;
 }
 
+static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
+{
+    if (obj == Py_None || obj == Py_True || obj == Py_False) {
+        /* don't increment the reference counter, Constant uses a borrowed
+         * reference, not a strong reference */
+        *out = obj;
+        return 0;
+    }
+
+    if (obj) {
+        if (PyArena_AddPyObject(arena, obj) < 0) {
+            *out = NULL;
+            return -1;
+        }
+        Py_INCREF(obj);
+    }
+    *out = obj;
+    return 0;
+}
+
 static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
 {
     if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
@@ -941,6 +966,8 @@ static int init_types(void)
     if (!NameConstant_type) return 0;
     Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
     if (!Ellipsis_type) return 0;
+    Constant_type = make_type("Constant", expr_type, Constant_fields, 1);
+    if (!Constant_type) return 0;
     Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
     if (!Attribute_type) return 0;
     Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3);
@@ -2167,6 +2194,25 @@ Ellipsis(int lineno, int col_offset, PyArena *arena)
 }
 
 expr_ty
+Constant(constant value, int lineno, int col_offset, PyArena *arena)
+{
+    expr_ty p;
+    if (!value) {
+        PyErr_SetString(PyExc_ValueError,
+                        "field value is required for Constant");
+        return NULL;
+    }
+    p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+    if (!p)
+        return NULL;
+    p->kind = Constant_kind;
+    p->v.Constant.value = value;
+    p->lineno = lineno;
+    p->col_offset = col_offset;
+    return p;
+}
+
+expr_ty
 Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
           col_offset, PyArena *arena)
 {
@@ -3267,6 +3313,15 @@ ast2obj_expr(void* _o)
         result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
         if (!result) goto failed;
         break;
+    case Constant_kind:
+        result = PyType_GenericNew(Constant_type, NULL, NULL);
+        if (!result) goto failed;
+        value = ast2obj_constant(o->v.Constant.value);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+            goto failed;
+        Py_DECREF(value);
+        break;
     case Attribute_kind:
         result = PyType_GenericNew(Attribute_type, NULL, NULL);
         if (!result) goto failed;
@@ -6240,6 +6295,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
+    isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type);
+    if (isinstance == -1) {
+        return 1;
+    }
+    if (isinstance) {
+        constant value;
+
+        if (_PyObject_HasAttrId(obj, &PyId_value)) {
+            int res;
+            tmp = _PyObject_GetAttrId(obj, &PyId_value);
+            if (tmp == NULL) goto failed;
+            res = obj2ast_constant(tmp, &value, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        } else {
+            PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant");
+            return 1;
+        }
+        *out = Constant(value, lineno, col_offset, arena);
+        if (*out == NULL) goto failed;
+        return 0;
+    }
     isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type);
     if (isinstance == -1) {
         return 1;
@@ -7517,6 +7594,8 @@ PyInit__ast(void)
         0) return NULL;
     if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
         return NULL;
+    if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0)
+        return NULL;
     if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0)
         return NULL;
     if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0)
diff --git a/Python/ast.c b/Python/ast.c
index a5d8dba..5422e9c 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -132,6 +132,50 @@ validate_arguments(arguments_ty args)
 }
 
 static int
+validate_constant(PyObject *value)
+{
+    if (value == Py_None || value == Py_Ellipsis)
+        return 1;
+
+    if (PyLong_CheckExact(value)
+            || PyFloat_CheckExact(value)
+            || PyComplex_CheckExact(value)
+            || PyBool_Check(value)
+            || PyUnicode_CheckExact(value)
+            || PyBytes_CheckExact(value))
+        return 1;
+
+    if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) {
+        PyObject *it;
+
+        it = PyObject_GetIter(value);
+        if (it == NULL)
+            return 0;
+
+        while (1) {
+            PyObject *item = PyIter_Next(it);
+            if (item == NULL) {
+                if (PyErr_Occurred()) {
+                    Py_DECREF(it);
+                    return 0;
+                }
+                break;
+            }
+
+            if (!validate_constant(item)) {
+                Py_DECREF(it);
+                return 0;
+            }
+        }
+
+        Py_DECREF(it);
+        return 1;
+    }
+
+    return 0;
+}
+
+static int
 validate_expr(expr_ty exp, expr_context_ty ctx)
 {
     int check_ctx = 1;
@@ -240,6 +284,12 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
         return validate_expr(exp->v.Call.func, Load) &&
             validate_exprs(exp->v.Call.args, Load, 0) &&
             validate_keywords(exp->v.Call.keywords);
+    case Constant_kind:
+        if (!validate_constant(exp->v.Constant.value)) {
+            PyErr_SetString(PyExc_TypeError, "invalid type in Constant");
+            return 0;
+        }
+        return 1;
     case Num_kind: {
         PyObject *n = exp->v.Num.n;
         if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) &&
diff --git a/Python/compile.c b/Python/compile.c
index a710e82..ccb05cf 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1314,7 +1314,11 @@ compiler_isdocstring(stmt_ty s)
 {
     if (s->kind != Expr_kind)
         return 0;
-    return s->v.Expr.value->kind == Str_kind;
+    if (s->v.Expr.value->kind == Str_kind)
+        return 1;
+    if (s->v.Expr.value->kind == Constant_kind)
+        return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
+    return 0;
 }
 
 /* Compile a sequence of statements, checking for a docstring. */
@@ -1688,8 +1692,12 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
 
     st = (stmt_ty)asdl_seq_GET(body, 0);
     docstring = compiler_isdocstring(st);
-    if (docstring && c->c_optimize < 2)
-        first_const = st->v.Expr.value->v.Str.s;
+    if (docstring && c->c_optimize < 2) {
+        if (st->v.Expr.value->kind == Constant_kind)
+            first_const = st->v.Expr.value->v.Constant.value;
+        else
+            first_const = st->v.Expr.value->v.Str.s;
+    }
     if (compiler_add_o(c, c->u->u_consts, first_const) < 0)      {
         compiler_exit_scope(c);
         return 0;
@@ -2600,6 +2608,36 @@ compiler_assert(struct compiler *c, stmt_ty s)
 }
 
 static int
+compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
+{
+    if (c->c_interactive && c->c_nestlevel <= 1) {
+        VISIT(c, expr, value);
+        ADDOP(c, PRINT_EXPR);
+        return 1;
+    }
+
+    if (value->kind == Str_kind || value->kind == Num_kind) {
+        /* ignore strings and numbers */
+        return 1;
+    }
+
+    if (value->kind == Constant_kind) {
+        PyObject *cst = value->v.Constant.value;
+        if (PyUnicode_CheckExact(cst)
+            || PyLong_CheckExact(cst)
+            || PyFloat_CheckExact(cst)
+            || PyComplex_CheckExact(cst)) {
+            /* ignore strings and numbers */
+            return 1;
+        }
+    }
+
+    VISIT(c, expr, value);
+    ADDOP(c, POP_TOP);
+    return 1;
+}
+
+static int
 compiler_visit_stmt(struct compiler *c, stmt_ty s)
 {
     Py_ssize_t i, n;
@@ -2669,16 +2707,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
     case Nonlocal_kind:
         break;
     case Expr_kind:
-        if (c->c_interactive && c->c_nestlevel <= 1) {
-            VISIT(c, expr, s->v.Expr.value);
-            ADDOP(c, PRINT_EXPR);
-        }
-        else if (s->v.Expr.value->kind != Str_kind &&
-                 s->v.Expr.value->kind != Num_kind) {
-            VISIT(c, expr, s->v.Expr.value);
-            ADDOP(c, POP_TOP);
-        }
-        break;
+        return compiler_visit_stmt_expr(c, s->v.Expr.value);
     case Pass_kind:
         break;
     case Break_kind:
@@ -3625,6 +3654,8 @@ expr_constant(struct compiler *c, expr_ty e)
     switch (e->kind) {
     case Ellipsis_kind:
         return 1;
+    case Constant_kind:
+        return PyObject_IsTrue(e->v.Constant.value);
     case Num_kind:
         return PyObject_IsTrue(e->v.Num.n);
     case Str_kind:
@@ -3912,6 +3943,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
         return compiler_compare(c, e);
     case Call_kind:
         return compiler_call(c, e);
+    case Constant_kind:
+        ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts);
+        break;
     case Num_kind:
         ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts);
         break;
diff --git a/Python/future.c b/Python/future.c
index 163f87f..75f2107 100644
--- a/Python/future.c
+++ b/Python/future.c
@@ -79,7 +79,10 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
 
     i = 0;
     first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
-    if (first->kind == Expr_kind && first->v.Expr.value->kind == Str_kind)
+    if (first->kind == Expr_kind
+        && (first->v.Expr.value->kind == Str_kind
+            || (first->v.Expr.value->kind == Constant_kind
+                && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
         i++;
 
 
diff --git a/Python/symtable.c b/Python/symtable.c
index 9f5149f..558bb06 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1455,6 +1455,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
     case JoinedStr_kind:
         VISIT_SEQ(st, expr, e->v.JoinedStr.values);
         break;
+    case Constant_kind:
     case Num_kind:
     case Str_kind:
     case Bytes_kind: