bpo-36817: Add f-string debugging using '='. (GH-13123)

If a "=" is specified a the end of an f-string expression, the f-string will evaluate to the text of the expression, followed by '=', followed by the repr of the value of the expression.

5 files changed, 142 insertions, 45 deletions

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 6c8488f..cb53a41 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -314,10 +314,12 @@ static char *Call_fields[]={
 static PyTypeObject *FormattedValue_type;
 _Py_IDENTIFIER(conversion);
 _Py_IDENTIFIER(format_spec);
+_Py_IDENTIFIER(expr_text);
 static char *FormattedValue_fields[]={
     "value",
     "conversion",
     "format_spec",
+    "expr_text",
 };
 static PyTypeObject *JoinedStr_type;
 static char *JoinedStr_fields[]={
@@ -950,7 +952,7 @@ static int init_types(void)
     Call_type = make_type("Call", expr_type, Call_fields, 3);
     if (!Call_type) return 0;
     FormattedValue_type = make_type("FormattedValue", expr_type,
-                                    FormattedValue_fields, 3);
+                                    FormattedValue_fields, 4);
     if (!FormattedValue_type) return 0;
     JoinedStr_type = make_type("JoinedStr", expr_type, JoinedStr_fields, 1);
     if (!JoinedStr_type) return 0;
@@ -2249,9 +2251,9 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
 }
 
 expr_ty
-FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
-               int col_offset, int end_lineno, int end_col_offset, PyArena
-               *arena)
+FormattedValue(expr_ty value, int conversion, expr_ty format_spec, string
+               expr_text, int lineno, int col_offset, int end_lineno, int
+               end_col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!value) {
@@ -2266,6 +2268,7 @@ FormattedValue(expr_ty value, int conversion, expr_ty format_spec, int lineno,
     p->v.FormattedValue.value = value;
     p->v.FormattedValue.conversion = conversion;
     p->v.FormattedValue.format_spec = format_spec;
+    p->v.FormattedValue.expr_text = expr_text;
     p->lineno = lineno;
     p->col_offset = col_offset;
     p->end_lineno = end_lineno;
@@ -3496,6 +3499,11 @@ ast2obj_expr(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_format_spec, value) == -1)
             goto failed;
         Py_DECREF(value);
+        value = ast2obj_string(o->v.FormattedValue.expr_text);
+        if (!value) goto failed;
+        if (_PyObject_SetAttrId(result, &PyId_expr_text, value) == -1)
+            goto failed;
+        Py_DECREF(value);
         break;
     case JoinedStr_kind:
         result = PyType_GenericNew(JoinedStr_type, NULL, NULL);
@@ -7148,6 +7156,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty value;
         int conversion;
         expr_ty format_spec;
+        string expr_text;
 
         if (_PyObject_LookupAttrId(obj, &PyId_value, &tmp) < 0) {
             return 1;
@@ -7188,8 +7197,22 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             if (res != 0) goto failed;
             Py_CLEAR(tmp);
         }
-        *out = FormattedValue(value, conversion, format_spec, lineno,
-                              col_offset, end_lineno, end_col_offset, arena);
+        if (_PyObject_LookupAttrId(obj, &PyId_expr_text, &tmp) < 0) {
+            return 1;
+        }
+        if (tmp == NULL || tmp == Py_None) {
+            Py_CLEAR(tmp);
+            expr_text = NULL;
+        }
+        else {
+            int res;
+            res = obj2ast_string(tmp, &expr_text, arena);
+            if (res != 0) goto failed;
+            Py_CLEAR(tmp);
+        }
+        *out = FormattedValue(value, conversion, format_spec, expr_text,
+                              lineno, col_offset, end_lineno, end_col_offset,
+                              arena);
         if (*out == NULL) goto failed;
         return 0;
     }
diff --git a/Python/ast.c b/Python/ast.c
index 4687f81..21abd7e 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -4854,7 +4854,8 @@ fstring_compile_expr(const char *expr_start, const char *expr_end,
 
     assert(expr_end >= expr_start);
     assert(*(expr_start-1) == '{');
-    assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':');
+    assert(*expr_end == '}' || *expr_end == '!' || *expr_end == ':' ||
+           *expr_end == '=');
 
     /* If the substring is all whitespace, it's an error.  We need to catch this
        here, and not when we call PyParser_SimpleParseStringFlagsFilename,
@@ -4997,9 +4998,9 @@ fstring_parse(const char **str, const char *end, int raw, int recurse_lvl,
               struct compiling *c, const node *n);
 
 /* Parse the f-string at *str, ending at end.  We know *str starts an
-   expression (so it must be a '{'). Returns the FormattedValue node,
-   which includes the expression, conversion character, and
-   format_spec expression.
+   expression (so it must be a '{'). Returns the FormattedValue node, which
+   includes the expression, conversion character, format_spec expression, and
+   optionally the text of the expression (if = is used).
 
    Note that I don't do a perfect job here: I don't make sure that a
    closing brace doesn't match an opening paren, for example. It
@@ -5016,7 +5017,12 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
     const char *expr_end;
     expr_ty simple_expression;
     expr_ty format_spec = NULL; /* Optional format specifier. */
-    int conversion = -1; /* The conversion char. -1 if not specified. */
+    int conversion = -1; /* The conversion char.  Use default if not
+                            specified, or !r if using = and no format
+                            spec. */
+    int equal_flag = 0; /* Are we using the = feature? */
+    PyObject *expr_text = NULL; /* The text of the expression, used for =. */
+    const char *expr_text_end;
 
     /* 0 if we're not in a string, else the quote char we're trying to
        match (single or double quote). */
@@ -5033,7 +5039,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
     /* Can only nest one level deep. */
     if (recurse_lvl >= 2) {
         ast_error(c, n, "f-string: expressions nested too deeply");
-        return -1;
+        goto error;
     }
 
     /* The first char must be a left brace, or we wouldn't have gotten
@@ -5061,7 +5067,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
             ast_error(c, n,
                       "f-string expression part "
                       "cannot include a backslash");
-            return -1;
+            goto error;
         }
         if (quote_char) {
             /* We're inside a string. See if we're at the end. */
@@ -5106,7 +5112,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
         } else if (ch == '[' || ch == '{' || ch == '(') {
             if (nested_depth >= MAXLEVEL) {
                 ast_error(c, n, "f-string: too many nested parenthesis");
-                return -1;
+                goto error;
             }
             parenstack[nested_depth] = ch;
             nested_depth++;
@@ -5114,22 +5120,38 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
             /* Error: can't include a comment character, inside parens
                or not. */
             ast_error(c, n, "f-string expression part cannot include '#'");
-            return -1;
+            goto error;
         } else if (nested_depth == 0 &&
-                   (ch == '!' || ch == ':' || ch == '}')) {
-            /* First, test for the special case of "!=". Since '=' is
-               not an allowed conversion character, nothing is lost in
-               this test. */
-            if (ch == '!' && *str+1 < end && *(*str+1) == '=') {
-                /* This isn't a conversion character, just continue. */
-                continue;
+                   (ch == '!' || ch == ':' || ch == '}' ||
+                    ch == '=' || ch == '>' || ch == '<')) {
+            /* See if there's a next character. */
+            if (*str+1 < end) {
+                char next = *(*str+1);
+
+                /* For "!=". since '=' is not an allowed conversion character,
+                   nothing is lost in this test. */
+                if ((ch == '!' && next == '=') ||   /* != */
+                    (ch == '=' && next == '=') ||   /* == */
+                    (ch == '<' && next == '=') ||   /* <= */
+                    (ch == '>' && next == '=')      /* >= */
+                    ) {
+                    *str += 1;
+                    continue;
+                }
+                /* Don't get out of the loop for these, if they're single
+                   chars (not part of 2-char tokens). If by themselves, they
+                   don't end an expression (unlike say '!'). */
+                if (ch == '>' || ch == '<') {
+                    continue;
+                }
             }
+
             /* Normal way out of this loop. */
             break;
         } else if (ch == ']' || ch == '}' || ch == ')') {
             if (!nested_depth) {
                 ast_error(c, n, "f-string: unmatched '%c'", ch);
-                return -1;
+                goto error;
             }
             nested_depth--;
             int opening = parenstack[nested_depth];
@@ -5141,7 +5163,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
                           "f-string: closing parenthesis '%c' "
                           "does not match opening parenthesis '%c'",
                           ch, opening);
-                return -1;
+                goto error;
             }
         } else {
             /* Just consume this char and loop around. */
@@ -5154,12 +5176,12 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
        let's just do that.*/
     if (quote_char) {
         ast_error(c, n, "f-string: unterminated string");
-        return -1;
+        goto error;
     }
     if (nested_depth) {
         int opening = parenstack[nested_depth - 1];
         ast_error(c, n, "f-string: unmatched '%c'", opening);
-        return -1;
+        goto error;
     }
 
     if (*str >= end)
@@ -5170,7 +5192,22 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
        conversion or format_spec. */
     simple_expression = fstring_compile_expr(expr_start, expr_end, c, n);
     if (!simple_expression)
-        return -1;
+        goto error;
+
+    /* Check for =, which puts the text value of the expression in
+       expr_text. */
+    if (**str == '=') {
+        *str += 1;
+        equal_flag = 1;
+
+        /* Skip over ASCII whitespace.  No need to test for end of string
+           here, since we know there's at least a trailing quote somewhere
+           ahead. */
+        while (Py_ISSPACE(**str)) {
+            *str += 1;
+        }
+        expr_text_end = *str;
+    }
 
     /* Check for a conversion char, if present. */
     if (**str == '!') {
@@ -5182,13 +5219,19 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
         *str += 1;
 
         /* Validate the conversion. */
-        if (!(conversion == 's' || conversion == 'r'
-              || conversion == 'a')) {
+        if (!(conversion == 's' || conversion == 'r' || conversion == 'a')) {
             ast_error(c, n,
                       "f-string: invalid conversion character: "
                       "expected 's', 'r', or 'a'");
-            return -1;
+            goto error;
         }
+
+    }
+    if (equal_flag) {
+        Py_ssize_t len = expr_text_end-expr_start;
+        expr_text = PyUnicode_FromStringAndSize(expr_start, len);
+        if (!expr_text)
+            goto error;
     }
 
     /* Check for the format spec, if present. */
@@ -5202,7 +5245,7 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
         /* Parse the format spec. */
         format_spec = fstring_parse(str, end, raw, recurse_lvl+1, c, n);
         if (!format_spec)
-            return -1;
+            goto error;
     }
 
     if (*str >= end || **str != '}')
@@ -5213,20 +5256,31 @@ fstring_find_expr(const char **str, const char *end, int raw, int recurse_lvl,
     assert(**str == '}');
     *str += 1;
 
+    /* If we're in = mode, and have no format spec and no explict conversion,
+       set the conversion to 'r'. */
+    if (equal_flag && format_spec == NULL && conversion == -1) {
+        conversion = 'r';
+    }
+
     /* And now create the FormattedValue node that represents this
        entire expression with the conversion and format spec. */
     *expression = FormattedValue(simple_expression, conversion,
-                                 format_spec, LINENO(n), n->n_col_offset,
-                                 n->n_end_lineno, n->n_end_col_offset,
-                                 c->c_arena);
+                                 format_spec, expr_text, LINENO(n),
+                                 n->n_col_offset, n->n_end_lineno,
+                                 n->n_end_col_offset, c->c_arena);
     if (!*expression)
-        return -1;
+        goto error;
 
     return 0;
 
 unexpected_end_of_string:
     ast_error(c, n, "f-string: expecting '}'");
+    /* Falls through to error. */
+
+error:
+    Py_XDECREF(expr_text);
     return -1;
+
 }
 
 /* Return -1 on error.
diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c
index 916ad5f..25a5c69 100644
--- a/Python/ast_unparse.c
+++ b/Python/ast_unparse.c
@@ -655,6 +655,11 @@ append_formattedvalue(_PyUnicodeWriter *writer, expr_ty e, bool is_format_spec)
     }
     Py_DECREF(temp_fv_str);
 
+    if (e->v.FormattedValue.expr_text) {
+        /* Use the = for debug text expansion. */
+        APPEND_STR("=");
+    }
+
     if (e->v.FormattedValue.conversion > 0) {
         switch (e->v.FormattedValue.conversion) {
         case 'a':
diff --git a/Python/ceval.c b/Python/ceval.c
index e616a3f..4e43df2 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -3435,13 +3435,15 @@ main_loop:
 
             /* See if any conversion is specified. */
             switch (which_conversion) {
+            case FVC_NONE:  conv_fn = NULL;           break;
             case FVC_STR:   conv_fn = PyObject_Str;   break;
             case FVC_REPR:  conv_fn = PyObject_Repr;  break;
             case FVC_ASCII: conv_fn = PyObject_ASCII; break;
-
-            /* Must be 0 (meaning no conversion), since only four
-               values are allowed by (oparg & FVC_MASK). */
-            default:        conv_fn = NULL;           break;
+            default:
+                PyErr_Format(PyExc_SystemError,
+                             "unexpected conversion flag %d",
+                             which_conversion);
+                goto error;
             }
 
             /* If there's a conversion function, call it and replace
diff --git a/Python/compile.c b/Python/compile.c
index 86f2a09..dd27ba8 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3946,8 +3946,8 @@ compiler_formatted_value(struct compiler *c, expr_ty e)
     /* Our oparg encodes 2 pieces of information: the conversion
        character, and whether or not a format_spec was provided.
 
-       Convert the conversion char to 2 bits:
-       None: 000  0x0  FVC_NONE
+       Convert the conversion char to 3 bits:
+           : 000  0x0  FVC_NONE   The default if nothing specified.
        !s  : 001  0x1  FVC_STR
        !r  : 010  0x2  FVC_REPR
        !a  : 011  0x3  FVC_ASCII
@@ -3957,19 +3957,26 @@ compiler_formatted_value(struct compiler *c, expr_ty e)
        no  : 000  0x0
     */
 
+    int conversion = e->v.FormattedValue.conversion;
     int oparg;
 
-    /* Evaluate the expression to be formatted. */
+    if (e->v.FormattedValue.expr_text) {
+        /* Push the text of the expression (which already has the '=' in
+           it. */
+        ADDOP_LOAD_CONST(c, e->v.FormattedValue.expr_text);
+    }
+
+    /* The expression to be formatted. */
     VISIT(c, expr, e->v.FormattedValue.value);
 
-    switch (e->v.FormattedValue.conversion) {
+    switch (conversion) {
     case 's': oparg = FVC_STR;   break;
     case 'r': oparg = FVC_REPR;  break;
     case 'a': oparg = FVC_ASCII; break;
     case -1:  oparg = FVC_NONE;  break;
     default:
-        PyErr_SetString(PyExc_SystemError,
-                        "Unrecognized conversion character");
+        PyErr_Format(PyExc_SystemError,
+                     "Unrecognized conversion character %d", conversion);
         return 0;
     }
     if (e->v.FormattedValue.format_spec) {
@@ -3980,6 +3987,12 @@ compiler_formatted_value(struct compiler *c, expr_ty e)
 
     /* And push our opcode and oparg */
     ADDOP_I(c, FORMAT_VALUE, oparg);
+
+    /* If we have expr_text, join the 2 strings on the stack. */
+    if (e->v.FormattedValue.expr_text) {
+        ADDOP_I(c, BUILD_STRING, 2);
+    }
+
     return 1;
 }