diff options
author | Evan Jones <ej@evanjones.ca> | 2012-01-05 13:28:54 (GMT) |
---|---|---|
committer | Evan Jones <ej@evanjones.ca> | 2012-01-05 15:12:41 (GMT) |
commit | 831775c7ecb5f4b9548a55a6f603e057fad3fea2 (patch) | |
tree | 94275311078d438c8604c42a78516c52b7e3201c | |
parent | 4b019b3cebf40ca9b1b95741f6336b843a9dad55 (diff) | |
download | Ninja-831775c7ecb5f4b9548a55a6f603e057fad3fea2.zip Ninja-831775c7ecb5f4b9548a55a6f603e057fad3fea2.tar.gz Ninja-831775c7ecb5f4b9548a55a6f603e057fad3fea2.tar.bz2 |
Lexer: include leading spaces in the newline token.
This means that indented blank lines are skipped without causing errors.
-rw-r--r-- | src/lexer.cc | 174 | ||||
-rw-r--r-- | src/lexer.in.cc | 2 | ||||
-rw-r--r-- | src/parsers_test.cc | 28 |
3 files changed, 119 insertions, 85 deletions
diff --git a/src/lexer.cc b/src/lexer.cc index b4707d2..5bc467b 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -201,6 +201,7 @@ yy5: { token = ERROR; break; } yy6: ++p; +yy7: { token = NEWLINE; break; } yy8: ++p; @@ -351,8 +352,13 @@ yy65: if (yybm[0+yych] & 128) { goto yy64; } + if (yych == '\n') goto yy66; if (yych == '#') goto yy59; goto yy3; +yy66: + ++p; + yych = *p; + goto yy7; } } @@ -415,39 +421,39 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yych <= ' ') { - if (yych <= 0x00) goto yy72; - if (yych <= 0x1F) goto yy74; + if (yych <= 0x00) goto yy73; + if (yych <= 0x1F) goto yy75; } else { - if (yych == '$') goto yy70; - goto yy74; + if (yych == '$') goto yy71; + goto yy75; } ++p; yych = *p; - goto yy78; -yy69: - { continue; } + goto yy79; yy70: - ++p; - if ((yych = *p) == '\n') goto yy75; + { continue; } yy71: - { break; } + ++p; + if ((yych = *p) == '\n') goto yy76; yy72: + { break; } +yy73: ++p; { break; } -yy74: - yych = *++p; - goto yy71; yy75: + yych = *++p; + goto yy72; +yy76: ++p; { continue; } -yy77: +yy78: ++p; yych = *p; -yy78: +yy79: if (yybm[0+yych] & 128) { - goto yy77; + goto yy78; } - goto yy69; + goto yy70; } } @@ -497,40 +503,40 @@ bool Lexer::ReadIdent(string* out) { yych = *p; if (yych <= '@') { if (yych <= '.') { - if (yych <= ',') goto yy83; + if (yych <= ',') goto yy84; } else { - if (yych <= '/') goto yy83; - if (yych >= ':') goto yy83; + if (yych <= '/') goto yy84; + if (yych >= ':') goto yy84; } } else { if (yych <= '_') { - if (yych <= 'Z') goto yy81; - if (yych <= '^') goto yy83; + if (yych <= 'Z') goto yy82; + if (yych <= '^') goto yy84; } else { - if (yych <= '`') goto yy83; - if (yych >= '{') goto yy83; + if (yych <= '`') goto yy84; + if (yych >= '{') goto yy84; } } -yy81: +yy82: ++p; yych = *p; - goto yy86; -yy82: + goto yy87; +yy83: { out->assign(start, p - start); break; } -yy83: +yy84: ++p; { return false; } -yy85: +yy86: ++p; yych = *p; -yy86: +yy87: if (yybm[0+yych] & 128) { - goto yy85; + goto yy86; } - goto yy82; + goto yy83; } } @@ -585,28 +591,28 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { yych = *p; if (yych <= '#') { if (yych <= '\n') { - if (yych <= 0x00) goto yy95; - if (yych >= '\n') goto yy91; + if (yych <= 0x00) goto yy96; + if (yych >= '\n') goto yy92; } else { - if (yych == ' ') goto yy91; + if (yych == ' ') goto yy92; } } else { if (yych <= ':') { - if (yych <= '$') goto yy93; - if (yych >= ':') goto yy91; + if (yych <= '$') goto yy94; + if (yych >= ':') goto yy92; } else { - if (yych == '|') goto yy91; + if (yych == '|') goto yy92; } } ++p; yych = *p; - goto yy117; -yy90: + goto yy118; +yy91: { eval->AddText(StringPiece(start, p - start)); continue; } -yy91: +yy92: ++p; { if (path) { @@ -619,36 +625,36 @@ yy91: continue; } } -yy93: +yy94: ++p; if ((yych = *p) <= '/') { if (yych <= ' ') { - if (yych == '\n') goto yy106; - if (yych <= 0x1F) goto yy97; - goto yy99; + if (yych == '\n') goto yy107; + if (yych <= 0x1F) goto yy98; + goto yy100; } else { if (yych <= '$') { - if (yych <= '#') goto yy97; - goto yy101; + if (yych <= '#') goto yy98; + goto yy102; } else { - if (yych == '-') goto yy103; - goto yy97; + if (yych == '-') goto yy104; + goto yy98; } } } else { if (yych <= '^') { - if (yych <= '9') goto yy103; - if (yych <= '@') goto yy97; - if (yych <= 'Z') goto yy103; - goto yy97; + if (yych <= '9') goto yy104; + if (yych <= '@') goto yy98; + if (yych <= 'Z') goto yy104; + goto yy98; } else { if (yych <= '`') { - if (yych <= '_') goto yy103; - goto yy97; + if (yych <= '_') goto yy104; + goto yy98; } else { - if (yych <= 'z') goto yy103; - if (yych <= '{') goto yy105; - goto yy97; + if (yych <= 'z') goto yy104; + if (yych <= '{') goto yy106; + goto yy98; } } } @@ -656,86 +662,86 @@ yy93: last_token_ = start; return Error("lexing error", err); } -yy95: +yy96: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy97: - ++p; yy98: + ++p; +yy99: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy99: +yy100: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy101: +yy102: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy103: +yy104: ++p; yych = *p; - goto yy115; -yy104: + goto yy116; +yy105: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy105: +yy106: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy109; + goto yy110; } - goto yy98; -yy106: + goto yy99; +yy107: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy106; + goto yy107; } { continue; } -yy109: +yy110: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy109; + goto yy110; } - if (yych == '}') goto yy112; + if (yych == '}') goto yy113; p = q; - goto yy98; -yy112: + goto yy99; +yy113: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy114: +yy115: ++p; yych = *p; -yy115: +yy116: if (yybm[0+yych] & 64) { - goto yy114; + goto yy115; } - goto yy104; -yy116: + goto yy105; +yy117: ++p; yych = *p; -yy117: +yy118: if (yybm[0+yych] & 128) { - goto yy116; + goto yy117; } - goto yy90; + goto yy91; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 954c364..cb7fa90 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -111,7 +111,7 @@ Lexer::Token Lexer::ReadToken() { varname = [a-zA-Z0-9_.-]+; [ ]*"#"[^\000\n]*"\n" { continue; } - [\n] { token = NEWLINE; break; } + [ ]*[\n] { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } "rule" { token = RULE; break; } diff --git a/src/parsers_test.cc b/src/parsers_test.cc index 356b880..f3ac517 100644 --- a/src/parsers_test.cc +++ b/src/parsers_test.cc @@ -82,6 +82,21 @@ TEST_F(ParserTest, IgnoreIndentedComments) { EXPECT_FALSE(rule->generator()); } +TEST_F(ParserTest, IgnoreIndentedBlankLines) { + // the indented blanks used to cause parse errors + ASSERT_NO_FATAL_FAILURE(AssertParse( +" \n" +"rule cat\n" +" command = cat $in > $out\n" +" \n" +"build result: cat in_1.cc in-2.O\n" +" \n" +"variable=1\n")); + + // the variable must be in the top level environment + EXPECT_EQ("1", state.bindings_.LookupVariable("variable")); +} + TEST_F(ParserTest, Variables) { ASSERT_NO_FATAL_FAILURE(AssertParse( "l = one-letter-test\n" @@ -466,6 +481,19 @@ TEST_F(ParserTest, Errors) { // as we see them, not after we've read them all! EXPECT_EQ("input:4: empty path\n", err); } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + // the indented blank line must terminate the rule + // this also verifies that "unexpected (token)" errors are correct + EXPECT_FALSE(parser.ParseTest("rule r\n" + " command = r\n" + " \n" + " generator = 1\n", &err)); + EXPECT_EQ("input:4: unexpected indent\n", err); + } } TEST_F(ParserTest, MissingInput) { |