From 4b499dd3fb49181efbd99f87e45ff923146cba8a Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 13 Feb 2003 22:07:59 +0000 Subject: - Finally fixed the bug in compile() and exec where a string ending with an indented code block but no newline would raise SyntaxError. This would have been a four-line change in parsetok.c... Except codeop.py depends on this behavior, so a compilation flag had to be invented that causes the tokenizer to revert to the old behavior; this required extra changes to 2 .h files, 2 .c files, and 2 .py files. (Fixes SF bug #501622.) --- Include/parsetok.h | 2 ++ Include/pythonrun.h | 1 + Lib/code.py | 3 ++- Lib/codeop.py | 9 +++++++-- Lib/test/test_codeop.py | 7 ++++--- Lib/test/test_compile.py | 9 +++++++++ Misc/NEWS | 8 ++++++++ Parser/parsetok.c | 9 +++++++++ Python/bltinmodule.c | 4 +++- Python/pythonrun.c | 8 ++------ 10 files changed, 47 insertions(+), 13 deletions(-) diff --git a/Include/parsetok.h b/Include/parsetok.h index 58ef76a..99a79b7 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -21,6 +21,8 @@ typedef struct { #define PyPARSE_YIELD_IS_KEYWORD 0x0001 #endif +#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 + PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int, perrdetail *); PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int, diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 87d7874..746ba7e 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -10,6 +10,7 @@ extern "C" { #define PyCF_MASK (CO_FUTURE_DIVISION) #define PyCF_MASK_OBSOLETE (CO_GENERATOR_ALLOWED | CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 +#define PyCF_DONT_IMPLY_DEDENT 0x0200 typedef struct { int cf_flags; /* bitmask of CO_xxx flags relevant to future */ diff --git a/Lib/code.py b/Lib/code.py index 75c64e6..6bdc658 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -303,4 +303,5 @@ def interact(banner=None, readfunc=None, local=None): if __name__ == '__main__': - interact() + import pdb + pdb.run("interact()\n") diff --git a/Lib/codeop.py b/Lib/codeop.py index c97b4df..cc9d5b2 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -63,6 +63,8 @@ _features = [getattr(__future__, fname) __all__ = ["compile_command", "Compile", "CommandCompiler"] +PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h + def _maybe_compile(compiler, source, filename, symbol): # Check for source consisting of only blank lines and comments for line in source.split("\n"): @@ -103,6 +105,9 @@ def _maybe_compile(compiler, source, filename, symbol): if not code1 and e1 == e2: raise SyntaxError, err1 +def _compile(source, filename, symbol): + return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) + def compile_command(source, filename="", symbol="single"): r"""Compile a command and determine whether it is incomplete. @@ -121,7 +126,7 @@ def compile_command(source, filename="", symbol="single"): syntax error (OverflowError and ValueError can be produced by malformed literals). """ - return _maybe_compile(compile, source, filename, symbol) + return _maybe_compile(_compile, source, filename, symbol) class Compile: """Instances of this class behave much like the built-in compile @@ -129,7 +134,7 @@ class Compile: statement, it "remembers" and compiles all subsequent program texts with the statement in force.""" def __init__(self): - self.flags = 0 + self.flags = PyCF_DONT_IMPLY_DEDENT def __call__(self, source, filename, symbol): codeob = compile(source, filename, symbol, self.flags, 1) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index a0ce0bc..7af83eb 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -5,13 +5,13 @@ import unittest from test.test_support import run_unittest -from codeop import compile_command +from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT class CodeopTests(unittest.TestCase): def assertValid(self, str, symbol='single'): '''succeed iff str is a valid piece of code''' - expected = compile(str, "", symbol) + expected = compile(str, "", symbol, PyCF_DONT_IMPLY_DEDENT) self.assertEquals( compile_command(str, "", symbol), expected) @@ -42,7 +42,8 @@ class CodeopTests(unittest.TestCase): # special case self.assertEquals(compile_command(""), - compile("pass", "", 'single')) + compile("pass", "", 'single', + PyCF_DONT_IMPLY_DEDENT)) av("3**3","eval") av("(lambda z: \n z**3)","eval") diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 1fe7f52..1812fbc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -89,6 +89,15 @@ expect_error("2.0e+") expect_error("1e-") expect_error("3-4e/21") +if verbose: + print "testing compile() of indented block w/o trailing newline" + +s = """ +if 1: + if 2: + pass""" +compile(s, "", "exec") + if verbose: print "testing literals with leading zeroes" diff --git a/Misc/NEWS b/Misc/NEWS index dc6f1db..f206985 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,14 @@ What's New in Python 2.3 alpha 2? Core and builtins ----------------- +- Finally fixed the bug in compile() and exec where a string ending + with an indented code block but no newline would raise SyntaxError. + This would have been a four-line change in parsetok.c... Except + codeop.py depends on this behavior, so a compilation flag had to be + invented that causes the tokenizer to revert to the old behavior; + this required extra changes to 2 .h files, 2 .c files, and 2 .py + files. + - If a new-style class defines neither __new__ nor __init__, its constructor would ignore all arguments. This is changed now: the constructor refuses arguments in this case. This might break code diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 82a5450..0b3e957 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -130,6 +130,15 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, if (type == ENDMARKER && started) { type = NEWLINE; /* Add an extra newline */ started = 0; + /* Add the right number of dedent tokens, + except if a certain flag is given -- + codeop.py uses this. */ + if (tok->indent && + !(flags & PyPARSE_DONT_IMPLY_DEDENT)) + { + tok->pendin = -tok->indent; + tok->indent = 0; + } } else started = 1; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 338e38d..62aa512 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -380,7 +380,9 @@ builtin_compile(PyObject *self, PyObject *args) return NULL; } - if (supplied_flags & ~(PyCF_MASK | PyCF_MASK_OBSOLETE)) { + if (supplied_flags & + ~(PyCF_MASK | PyCF_MASK_OBSOLETE | PyCF_DONT_IMPLY_DEDENT)) + { PyErr_SetString(PyExc_ValueError, "compile(): unrecognised flags"); return NULL; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index fa72fe8..1faab50 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -548,13 +548,9 @@ PyRun_InteractiveOne(FILE *fp, const char *filename) } /* compute parser flags based on compiler flags */ -#if 0 /* future keyword */ #define PARSER_FLAGS(flags) \ - (((flags) && (flags)->cf_flags & CO_GENERATOR_ALLOWED) ? \ - PyPARSE_YIELD_IS_KEYWORD : 0) -#else -#define PARSER_FLAGS(flags) 0 -#endif + (((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \ + PyPARSE_DONT_IMPLY_DEDENT : 0) int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) -- cgit v0.12