From 3baedf17677e5d8f0c248d950e0681a0f5668992 Mon Sep 17 00:00:00 2001 From: Thiago Farina Date: Mon, 9 Jul 2012 16:09:31 -0300 Subject: Rename parsers.* to manifest_parser.* So it matches with the class name in there. Signed-off-by: Thiago Farina --- configure.py | 4 +- misc/ninja.vim | 6 +- src/build_log_perftest.cc | 2 +- src/graph.cc | 2 +- src/manifest_parser.cc | 327 +++++++++++++++++++++ src/manifest_parser.h | 71 +++++ src/manifest_parser_test.cc | 684 ++++++++++++++++++++++++++++++++++++++++++++ src/ninja.cc | 2 +- src/parsers.cc | 327 --------------------- src/parsers.h | 71 ----- src/parsers_test.cc | 684 -------------------------------------------- src/test.cc | 2 +- 12 files changed, 1091 insertions(+), 1091 deletions(-) create mode 100644 src/manifest_parser.cc create mode 100644 src/manifest_parser.h create mode 100644 src/manifest_parser_test.cc delete mode 100644 src/parsers.cc delete mode 100644 src/parsers.h delete mode 100644 src/parsers_test.cc diff --git a/configure.py b/configure.py index 5329708..6fb718f 100755 --- a/configure.py +++ b/configure.py @@ -236,8 +236,8 @@ for name in ['build', 'graph', 'graphviz', 'lexer', + 'manifest_parser', 'metrics', - 'parsers', 'state', 'util']: objs += cxx(name) @@ -305,7 +305,7 @@ for name in ['build_log_test', 'edit_distance_test', 'graph_test', 'lexer_test', - 'parsers_test', + 'manifest_parser_test', 'state_test', 'subprocess_test', 'test', diff --git a/misc/ninja.vim b/misc/ninja.vim index 51a95c2..6f0e48d 100644 --- a/misc/ninja.vim +++ b/misc/ninja.vim @@ -10,7 +10,7 @@ " ninja lexer and parser are at " https://github.com/martine/ninja/blob/master/src/lexer.in.cc -" https://github.com/martine/ninja/blob/master/src/parsers.cc +" https://github.com/martine/ninja/blob/master/src/manifest_parser.cc if exists("b:current_syntax") finish @@ -22,7 +22,7 @@ syn match ninjaComment /#.*/ contains=@Spell " Toplevel statements are the ones listed here and " toplevel variable assignments (ident '=' value). -" lexer.in.cc, ReadToken() and parsers.cc, Parse() +" lexer.in.cc, ReadToken() and manifest_parser.cc, Parse() syn match ninjaKeyword "^build\>" syn match ninjaKeyword "^rule\>" syn match ninjaKeyword "^default\>" @@ -33,7 +33,7 @@ syn match ninjaKeyword "^subninja\>" " on the first line without indent. 'rule' allows only a " limited set of magic variables, 'build' allows general " let assignments. -" parsers.cc, ParseRule() +" manifest_parser.cc, ParseRule() syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent syn keyword ninjaRuleCommand contained command depfile description generator restat diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index 51bb51b..02f4c60 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -17,7 +17,7 @@ #include "build_log.h" #include "graph.h" -#include "parsers.h" +#include "manifest_parser.h" #include "state.h" #include "util.h" diff --git a/src/graph.cc b/src/graph.cc index 56584e3..071a3b6 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -21,8 +21,8 @@ #include "depfile_parser.h" #include "disk_interface.h" #include "explain.h" +#include "manifest_parser.h" #include "metrics.h" -#include "parsers.h" #include "state.h" #include "util.h" diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc new file mode 100644 index 0000000..057e12c --- /dev/null +++ b/src/manifest_parser.cc @@ -0,0 +1,327 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "manifest_parser.h" + +#include +#include +#include +#include + +#include "graph.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +ManifestParser::ManifestParser(State* state, FileReader* file_reader) + : state_(state), file_reader_(file_reader) { + env_ = &state->bindings_; +} +bool ManifestParser::Load(const string& filename, string* err) { + string contents; + string read_err; + if (!file_reader_->ReadFile(filename, &contents, &read_err)) { + *err = "loading '" + filename + "': " + read_err; + return false; + } + contents.resize(contents.size() + 10); + return Parse(filename, contents, err); +} + +bool ManifestParser::Parse(const string& filename, const string& input, + string* err) { + METRIC_RECORD(".ninja parse"); + lexer_.Start(filename, input); + + for (;;) { + Lexer::Token token = lexer_.ReadToken(); + switch (token) { + case Lexer::BUILD: + if (!ParseEdge(err)) + return false; + break; + case Lexer::RULE: + if (!ParseRule(err)) + return false; + break; + case Lexer::DEFAULT: + if (!ParseDefault(err)) + return false; + break; + case Lexer::IDENT: { + lexer_.UnreadToken(); + string name; + EvalString value; + if (!ParseLet(&name, &value, err)) + return false; + env_->AddBinding(name, value.Evaluate(env_)); + break; + } + case Lexer::INCLUDE: + if (!ParseFileInclude(false, err)) + return false; + break; + case Lexer::SUBNINJA: + if (!ParseFileInclude(true, err)) + return false; + break; + case Lexer::ERROR: + return lexer_.Error("lexing error", err); + case Lexer::TEOF: + return true; + case Lexer::NEWLINE: + break; + default: + return lexer_.Error(string("unexpected ") + Lexer::TokenName(token), + err); + } + } + return false; // not reached +} + +bool ManifestParser::ParseRule(string* err) { + string name; + if (!lexer_.ReadIdent(&name)) + return lexer_.Error("expected rule name", err); + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + if (state_->LookupRule(name) != NULL) { + *err = "duplicate rule '" + name + "'"; + return false; + } + + Rule* rule = new Rule(name); // XXX scoped_ptr + + while (lexer_.PeekToken(Lexer::INDENT)) { + string key; + EvalString value; + if (!ParseLet(&key, &value, err)) + return false; + + if (key == "command") { + rule->command_ = value; + } else if (key == "depfile") { + rule->depfile_ = value; + } else if (key == "description") { + rule->description_ = value; + } else if (key == "generator") { + rule->generator_ = true; + } else if (key == "restat") { + rule->restat_ = true; + } else if (key == "rspfile") { + rule->rspfile_ = value; + } else if (key == "rspfile_content") { + rule->rspfile_content_ = value; + } else { + // Die on other keyvals for now; revisit if we want to add a + // scope here. + return lexer_.Error("unexpected variable '" + key + "'", err); + } + } + + if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) + return lexer_.Error("rspfile and rspfile_content need to be both specified", err); + + if (rule->command_.empty()) + return lexer_.Error("expected 'command =' line", err); + + state_->AddRule(rule); + return true; +} + +bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) { + if (!lexer_.ReadIdent(key)) + return false; + if (!ExpectToken(Lexer::EQUALS, err)) + return false; + if (!lexer_.ReadVarValue(value, err)) + return false; + return true; +} + +bool ManifestParser::ParseDefault(string* err) { + EvalString eval; + if (!lexer_.ReadPath(&eval, err)) + return false; + if (eval.empty()) + return lexer_.Error("expected target name", err); + + do { + string path = eval.Evaluate(env_); + string path_err; + if (!CanonicalizePath(&path, &path_err)) + return lexer_.Error(path_err, err); + if (!state_->AddDefault(path, &path_err)) + return lexer_.Error(path_err, err); + + eval.Clear(); + if (!lexer_.ReadPath(&eval, err)) + return false; + } while (!eval.empty()); + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + return true; +} + +bool ManifestParser::ParseEdge(string* err) { + vector ins, outs; + + { + EvalString out; + if (!lexer_.ReadPath(&out, err)) + return false; + if (out.empty()) + return lexer_.Error("expected path", err); + + do { + outs.push_back(out); + + out.Clear(); + if (!lexer_.ReadPath(&out, err)) + return false; + } while (!out.empty()); + } + + if (!ExpectToken(Lexer::COLON, err)) + return false; + + string rule_name; + if (!lexer_.ReadIdent(&rule_name)) + return lexer_.Error("expected build command name", err); + + const Rule* rule = state_->LookupRule(rule_name); + if (!rule) + return lexer_.Error("unknown build rule '" + rule_name + "'", err); + + for (;;) { + // XXX should we require one path here? + EvalString in; + if (!lexer_.ReadPath(&in, err)) + return false; + if (in.empty()) + break; + ins.push_back(in); + } + + // Add all implicit deps, counting how many as we go. + int implicit = 0; + if (lexer_.PeekToken(Lexer::PIPE)) { + for (;;) { + EvalString in; + if (!lexer_.ReadPath(&in, err)) + return err; + if (in.empty()) + break; + ins.push_back(in); + ++implicit; + } + } + + // Add all order-only deps, counting how many as we go. + int order_only = 0; + if (lexer_.PeekToken(Lexer::PIPE2)) { + for (;;) { + EvalString in; + if (!lexer_.ReadPath(&in, err)) + return false; + if (in.empty()) + break; + ins.push_back(in); + ++order_only; + } + } + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + // Default to using outer env. + BindingEnv* env = env_; + + // But create and fill a nested env if there are variables in scope. + if (lexer_.PeekToken(Lexer::INDENT)) { + // XXX scoped_ptr to handle error case. + env = new BindingEnv(env_); + do { + string key; + EvalString val; + if (!ParseLet(&key, &val, err)) + return false; + env->AddBinding(key, val.Evaluate(env_)); + } while (lexer_.PeekToken(Lexer::INDENT)); + } + + Edge* edge = state_->AddEdge(rule); + edge->env_ = env; + for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { + string path = i->Evaluate(env); + string path_err; + if (!CanonicalizePath(&path, &path_err)) + return lexer_.Error(path_err, err); + state_->AddIn(edge, path); + } + for (vector::iterator i = outs.begin(); i != outs.end(); ++i) { + string path = i->Evaluate(env); + string path_err; + if (!CanonicalizePath(&path, &path_err)) + return lexer_.Error(path_err, err); + state_->AddOut(edge, path); + } + edge->implicit_deps_ = implicit; + edge->order_only_deps_ = order_only; + + return true; +} + +bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { + // XXX this should use ReadPath! + EvalString eval; + if (!lexer_.ReadPath(&eval, err)) + return false; + string path = eval.Evaluate(env_); + + string contents; + string read_err; + if (!file_reader_->ReadFile(path, &contents, &read_err)) + return lexer_.Error("loading '" + path + "': " + read_err, err); + + ManifestParser subparser(state_, file_reader_); + if (new_scope) { + subparser.env_ = new BindingEnv(env_); + } else { + subparser.env_ = env_; + } + + if (!subparser.Parse(path, contents, err)) + return false; + + if (!ExpectToken(Lexer::NEWLINE, err)) + return false; + + return true; +} + +bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) { + Lexer::Token token = lexer_.ReadToken(); + if (token != expected) { + string message = string("expected ") + Lexer::TokenName(expected); + message += string(", got ") + Lexer::TokenName(token); + message += Lexer::TokenErrorHint(expected); + return lexer_.Error(message, err); + } + return true; +} diff --git a/src/manifest_parser.h b/src/manifest_parser.h new file mode 100644 index 0000000..a2c6c93 --- /dev/null +++ b/src/manifest_parser.h @@ -0,0 +1,71 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_MANIFEST_PARSER_H_ +#define NINJA_MANIFEST_PARSER_H_ + +#include +#include +#include + +using namespace std; + +#include "lexer.h" +#include "string_piece.h" + +struct BindingEnv; +struct EvalString; +struct State; + +/// Parses .ninja files. +struct ManifestParser { + struct FileReader { + virtual ~FileReader() {} + virtual bool ReadFile(const string& path, string* content, string* err) = 0; + }; + + ManifestParser(State* state, FileReader* file_reader); + + /// Load and parse a file. + bool Load(const string& filename, string* err); + + /// Parse a text string of input. Used by tests. + bool ParseTest(const string& input, string* err) { + return Parse("input", input, err); + } + +private: + /// Parse a file, given its contents as a string. + bool Parse(const string& filename, const string& input, string* err); + + /// Parse various statement types. + bool ParseRule(string* err); + bool ParseLet(string* key, EvalString* val, string* err); + bool ParseEdge(string* err); + bool ParseDefault(string* err); + + /// Parse either a 'subninja' or 'include' line. + bool ParseFileInclude(bool new_scope, string* err); + + /// If the next token is not \a expected, produce an error string + /// saying "expectd foo, got bar". + bool ExpectToken(Lexer::Token expected, string* err); + + State* state_; + BindingEnv* env_; + FileReader* file_reader_; + Lexer lexer_; +}; + +#endif // NINJA_MANIFEST_PARSER_H_ diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc new file mode 100644 index 0000000..9c6644c --- /dev/null +++ b/src/manifest_parser_test.cc @@ -0,0 +1,684 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "manifest_parser.h" + +#include + +#include "graph.h" +#include "state.h" + +struct ParserTest : public testing::Test, + public ManifestParser::FileReader { + void AssertParse(const char* input) { + ManifestParser parser(&state, this); + string err; + ASSERT_TRUE(parser.ParseTest(input, &err)) << err; + ASSERT_EQ("", err); + } + + virtual bool ReadFile(const string& path, string* content, string* err) { + files_read_.push_back(path); + map::iterator i = files_.find(path); + if (i == files_.end()) { + *err = "No such file or directory"; // Match strerror() for ENOENT. + return false; + } + *content = i->second; + return true; + } + + State state; + map files_; + vector files_read_; +}; + +TEST_F(ParserTest, Empty) { + ASSERT_NO_FATAL_FAILURE(AssertParse("")); +} + +TEST_F(ParserTest, Rules) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"\n" +"rule date\n" +" command = date > $out\n" +"\n" +"build result: cat in_1.cc in-2.O\n")); + + ASSERT_EQ(3u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("cat", rule->name()); + EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize()); +} + +TEST_F(ParserTest, IgnoreIndentedComments) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +" #indented comment\n" +"rule cat\n" +" command = cat $in > $out\n" +" #generator = 1\n" +" restat = 1 # comment\n" +" #comment\n" +"build result: cat in_1.cc in-2.O\n" +" #comment\n")); + + ASSERT_EQ(2u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("cat", rule->name()); + EXPECT_TRUE(rule->restat()); + EXPECT_FALSE(rule->generator()); +} + +TEST_F(ParserTest, IgnoreIndentedBlankLines) { + // the indented blanks used to cause parse errors + ASSERT_NO_FATAL_FAILURE(AssertParse( +" \n" +"rule cat\n" +" command = cat $in > $out\n" +" \n" +"build result: cat in_1.cc in-2.O\n" +" \n" +"variable=1\n")); + + // the variable must be in the top level environment + EXPECT_EQ("1", state.bindings_.LookupVariable("variable")); +} + +TEST_F(ParserTest, ResponseFiles) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat_rsp\n" +" command = cat $rspfile > $out\n" +" rspfile = $rspfile\n" +" rspfile_content = $in\n" +"\n" +"build out: cat_rsp in\n" +" rspfile=out.rsp\n")); + + ASSERT_EQ(2u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("cat_rsp", rule->name()); + EXPECT_EQ("[cat ][$rspfile][ > ][$out]", rule->command().Serialize()); + EXPECT_EQ("[$rspfile]", rule->rspfile().Serialize()); + EXPECT_EQ("[$in]", rule->rspfile_content().Serialize()); +} + +TEST_F(ParserTest, InNewline) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat_rsp\n" +" command = cat $in_newline > $out\n" +"\n" +"build out: cat_rsp in in2\n" +" rspfile=out.rsp\n")); + + ASSERT_EQ(2u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("cat_rsp", rule->name()); + EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->command().Serialize()); + + Edge* edge = state.edges_[0]; + EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand()); +} + +TEST_F(ParserTest, Variables) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"l = one-letter-test\n" +"rule link\n" +" command = ld $l $extra $with_under -o $out $in\n" +"\n" +"extra = -pthread\n" +"with_under = -under\n" +"build a: link b c\n" +"nested1 = 1\n" +"nested2 = $nested1/2\n" +"build supernested: link x\n" +" extra = $nested2/3\n")); + + ASSERT_EQ(2u, state.edges_.size()); + Edge* edge = state.edges_[0]; + EXPECT_EQ("ld one-letter-test -pthread -under -o a b c", + edge->EvaluateCommand()); + EXPECT_EQ("1/2", state.bindings_.LookupVariable("nested2")); + + edge = state.edges_[1]; + EXPECT_EQ("ld one-letter-test 1/2/3 -under -o supernested x", + edge->EvaluateCommand()); +} + +TEST_F(ParserTest, VariableScope) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"foo = bar\n" +"rule cmd\n" +" command = cmd $foo $in $out\n" +"\n" +"build inner: cmd a\n" +" foo = baz\n" +"build outer: cmd b\n" +"\n" // Extra newline after build line tickles a regression. +)); + + ASSERT_EQ(2u, state.edges_.size()); + EXPECT_EQ("cmd baz a inner", state.edges_[0]->EvaluateCommand()); + EXPECT_EQ("cmd bar b outer", state.edges_[1]->EvaluateCommand()); +} + +TEST_F(ParserTest, Continuation) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule link\n" +" command = foo bar $\n" +" baz\n" +"\n" +"build a: link c $\n" +" d e f\n")); + + ASSERT_EQ(2u, state.rules_.size()); + const Rule* rule = state.rules_.begin()->second; + EXPECT_EQ("link", rule->name()); + EXPECT_EQ("[foo bar baz]", rule->command().Serialize()); +} + +TEST_F(ParserTest, Backslash) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"foo = bar\\baz\n" +"foo2 = bar\\ baz\n" +)); + EXPECT_EQ("bar\\baz", state.bindings_.LookupVariable("foo")); + EXPECT_EQ("bar\\ baz", state.bindings_.LookupVariable("foo2")); +} + +TEST_F(ParserTest, Comment) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"# this is a comment\n" +"foo = not # a comment\n")); + EXPECT_EQ("not # a comment", state.bindings_.LookupVariable("foo")); +} + +TEST_F(ParserTest, Dollars) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule foo\n" +" command = ${out}bar$$baz$$$\n" +"blah\n" +"x = $$dollar\n" +"build $x: foo y\n" +)); + EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x")); + EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand()); +} + +TEST_F(ParserTest, EscapeSpaces) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule spaces\n" +" command = something\n" +"build foo$ bar: spaces $$one two$$$ three\n" +)); + EXPECT_TRUE(state.LookupNode("foo bar")); + EXPECT_EQ(state.edges_[0]->outputs_[0]->path(), "foo bar"); + EXPECT_EQ(state.edges_[0]->inputs_[0]->path(), "$one"); + EXPECT_EQ(state.edges_[0]->inputs_[1]->path(), "two$ three"); + EXPECT_EQ(state.edges_[0]->EvaluateCommand(), "something"); +} + +TEST_F(ParserTest, CanonicalizeFile) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build out: cat in/1 in//2\n" +"build in/1: cat\n" +"build in/2: cat\n")); + + EXPECT_TRUE(state.LookupNode("in/1")); + EXPECT_TRUE(state.LookupNode("in/2")); + EXPECT_FALSE(state.LookupNode("in//1")); + EXPECT_FALSE(state.LookupNode("in//2")); +} + +TEST_F(ParserTest, PathVariables) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"dir = out\n" +"build $dir/exe: cat src\n")); + + EXPECT_FALSE(state.LookupNode("$dir/exe")); + EXPECT_TRUE(state.LookupNode("out/exe")); +} + +TEST_F(ParserTest, CanonicalizePaths) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build ./out.o: cat ./bar/baz/../foo.cc\n")); + + EXPECT_FALSE(state.LookupNode("./out.o")); + EXPECT_TRUE(state.LookupNode("out.o")); + EXPECT_FALSE(state.LookupNode("./bar/baz/../foo.cc")); + EXPECT_TRUE(state.LookupNode("bar/foo.cc")); +} + +TEST_F(ParserTest, ReservedWords) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule build\n" +" command = rule run $out\n" +"build subninja: build include default foo.cc\n" +"default subninja\n")); +} + +TEST_F(ParserTest, Errors) { + { + ManifestParser parser(NULL, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("foobar", &err)); + EXPECT_EQ("input:1: expected '=', got eof\n" + "foobar\n" + " ^ near here" + , err); + } + + { + ManifestParser parser(NULL, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x 3", &err)); + EXPECT_EQ("input:1: expected '=', got identifier\n" + "x 3\n" + " ^ near here" + , err); + } + + { + ManifestParser parser(NULL, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x = 3", &err)); + EXPECT_EQ("input:1: unexpected EOF\n" + "x = 3\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x = 3\ny 2", &err)); + EXPECT_EQ("input:2: expected '=', got identifier\n" + "y 2\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x = $", &err)); + EXPECT_EQ("input:1: bad $-escape (literal $ must be written as $$)\n" + "x = $\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x = $\n $[\n", &err)); + EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n" + " $[\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("x = a$\n b$\n $\n", &err)); + EXPECT_EQ("input:4: unexpected EOF\n" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err)); + EXPECT_EQ("input:1: unknown build rule 'y'\n" + "build x: y z\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("build x:: y z\n", &err)); + EXPECT_EQ("input:1: expected build command name\n" + "build x:: y z\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n command = cat ok\n" + "build x: cat $\n :\n", + &err)); + EXPECT_EQ("input:4: expected newline, got ':'\n" + " :\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n", + &err)); + EXPECT_EQ("input:2: expected 'command =' line\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = ${fafsd\n" + "foo = bar\n", + &err)); + EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n" + " command = ${fafsd\n" + " ^ near here" + , err); + } + + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = cat\n" + "build $.: cat foo\n", + &err)); + EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n" + "build $.: cat foo\n" + " ^ near here" + , err); + } + + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cat\n" + " command = cat\n" + "build $: cat foo\n", + &err)); + EXPECT_EQ("input:3: expected ':', got newline ($ also escapes ':')\n" + "build $: cat foo\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule %foo\n", + &err)); + EXPECT_EQ("input:1: expected rule name\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n" + " command = foo\n" + " othervar = bar\n", + &err)); + EXPECT_EQ("input:3: unexpected variable 'othervar'\n" + " othervar = bar\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" + "build $.: cc bar.cc\n", + &err)); + EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n" + "build $.: cc bar.cc\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" + "build $: cc bar.cc\n", + &err)); + EXPECT_EQ("input:3: expected ':', got newline ($ also escapes ':')\n" + "build $: cc bar.cc\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("default\n", + &err)); + EXPECT_EQ("input:1: expected target name\n" + "default\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("default nonexistent\n", + &err)); + EXPECT_EQ("input:1: unknown target 'nonexistent'\n" + "default nonexistent\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule r\n command = r\n" + "build b: r\n" + "default b:\n", + &err)); + EXPECT_EQ("input:4: expected newline, got ':'\n" + "default b:\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("default $a\n", &err)); + EXPECT_EQ("input:1: empty path\n" + "default $a\n" + " ^ near here" + , err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_FALSE(parser.ParseTest("rule r\n" + " command = r\n" + "build $a: r $c\n", &err)); + // XXX the line number is wrong; we should evaluate paths in ParseEdge + // as we see them, not after we've read them all! + EXPECT_EQ("input:4: empty path\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; + // the indented blank line must terminate the rule + // this also verifies that "unexpected (token)" errors are correct + EXPECT_FALSE(parser.ParseTest("rule r\n" + " command = r\n" + " \n" + " generator = 1\n", &err)); + EXPECT_EQ("input:4: unexpected indent\n", err); + } +} + +TEST_F(ParserTest, MissingInput) { + State state; + ManifestParser parser(&state, this); + string err; + EXPECT_FALSE(parser.Load("build.ninja", &err)); + EXPECT_EQ("loading 'build.ninja': No such file or directory", err); +} + +TEST_F(ParserTest, MultipleOutputs) { + State state; + ManifestParser parser(&state, NULL); + string err; + EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n depfile = bar\n" + "build a.o b.o: cc c.cc\n", + &err)); + EXPECT_EQ("", err); +} + +TEST_F(ParserTest, SubNinja) { + files_["test.ninja"] = + "var = inner\n" + "build $builddir/inner: varref\n"; + ASSERT_NO_FATAL_FAILURE(AssertParse( +"builddir = some_dir/\n" +"rule varref\n" +" command = varref $var\n" +"var = outer\n" +"build $builddir/outer: varref\n" +"subninja test.ninja\n" +"build $builddir/outer2: varref\n")); + ASSERT_EQ(1u, files_read_.size()); + + EXPECT_EQ("test.ninja", files_read_[0]); + EXPECT_TRUE(state.LookupNode("some_dir/outer")); + // Verify our builddir setting is inherited. + EXPECT_TRUE(state.LookupNode("some_dir/inner")); + + ASSERT_EQ(3u, state.edges_.size()); + EXPECT_EQ("varref outer", state.edges_[0]->EvaluateCommand()); + EXPECT_EQ("varref inner", state.edges_[1]->EvaluateCommand()); + EXPECT_EQ("varref outer", state.edges_[2]->EvaluateCommand()); +} + +TEST_F(ParserTest, MissingSubNinja) { + ManifestParser parser(&state, this); + string err; + EXPECT_FALSE(parser.ParseTest("subninja foo.ninja\n", &err)); + EXPECT_EQ("input:1: loading 'foo.ninja': No such file or directory\n" + "subninja foo.ninja\n" + " ^ near here" + , err); +} + +TEST_F(ParserTest, Include) { + files_["include.ninja"] = "var = inner\n"; + ASSERT_NO_FATAL_FAILURE(AssertParse( +"var = outer\n" +"include include.ninja\n")); + + ASSERT_EQ(1u, files_read_.size()); + EXPECT_EQ("include.ninja", files_read_[0]); + EXPECT_EQ("inner", state.bindings_.LookupVariable("var")); +} + +TEST_F(ParserTest, Implicit) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build foo: cat bar | baz\n")); + + Edge* edge = state.LookupNode("foo")->in_edge(); + ASSERT_TRUE(edge->is_implicit(1)); +} + +TEST_F(ParserTest, OrderOnly) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build foo: cat bar || baz\n")); + + Edge* edge = state.LookupNode("foo")->in_edge(); + ASSERT_TRUE(edge->is_order_only(1)); +} + +TEST_F(ParserTest, DefaultDefault) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build a: cat foo\n" +"build b: cat foo\n" +"build c: cat foo\n" +"build d: cat foo\n")); + + string err; + EXPECT_EQ(4u, state.DefaultNodes(&err).size()); + EXPECT_EQ("", err); +} + +TEST_F(ParserTest, DefaultStatements) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build a: cat foo\n" +"build b: cat foo\n" +"build c: cat foo\n" +"build d: cat foo\n" +"third = c\n" +"default a b\n" +"default $third\n")); + + string err; + std::vector nodes = state.DefaultNodes(&err); + EXPECT_EQ("", err); + ASSERT_EQ(3u, nodes.size()); + EXPECT_EQ("a", nodes[0]->path()); + EXPECT_EQ("b", nodes[1]->path()); + EXPECT_EQ("c", nodes[2]->path()); +} + +TEST_F(ParserTest, UTF8) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule utf8\n" +" command = true\n" +" description = compilaci\xC3\xB3\n")); +} diff --git a/src/ninja.cc b/src/ninja.cc index 6090bdb..809768c 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -42,8 +42,8 @@ #include "explain.h" #include "graph.h" #include "graphviz.h" +#include "manifest_parser.h" #include "metrics.h" -#include "parsers.h" #include "state.h" #include "util.h" diff --git a/src/parsers.cc b/src/parsers.cc deleted file mode 100644 index bc76ba1..0000000 --- a/src/parsers.cc +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "parsers.h" - -#include -#include -#include -#include - -#include "graph.h" -#include "metrics.h" -#include "state.h" -#include "util.h" - -ManifestParser::ManifestParser(State* state, FileReader* file_reader) - : state_(state), file_reader_(file_reader) { - env_ = &state->bindings_; -} -bool ManifestParser::Load(const string& filename, string* err) { - string contents; - string read_err; - if (!file_reader_->ReadFile(filename, &contents, &read_err)) { - *err = "loading '" + filename + "': " + read_err; - return false; - } - contents.resize(contents.size() + 10); - return Parse(filename, contents, err); -} - -bool ManifestParser::Parse(const string& filename, const string& input, - string* err) { - METRIC_RECORD(".ninja parse"); - lexer_.Start(filename, input); - - for (;;) { - Lexer::Token token = lexer_.ReadToken(); - switch (token) { - case Lexer::BUILD: - if (!ParseEdge(err)) - return false; - break; - case Lexer::RULE: - if (!ParseRule(err)) - return false; - break; - case Lexer::DEFAULT: - if (!ParseDefault(err)) - return false; - break; - case Lexer::IDENT: { - lexer_.UnreadToken(); - string name; - EvalString value; - if (!ParseLet(&name, &value, err)) - return false; - env_->AddBinding(name, value.Evaluate(env_)); - break; - } - case Lexer::INCLUDE: - if (!ParseFileInclude(false, err)) - return false; - break; - case Lexer::SUBNINJA: - if (!ParseFileInclude(true, err)) - return false; - break; - case Lexer::ERROR: - return lexer_.Error("lexing error", err); - case Lexer::TEOF: - return true; - case Lexer::NEWLINE: - break; - default: - return lexer_.Error(string("unexpected ") + Lexer::TokenName(token), - err); - } - } - return false; // not reached -} - -bool ManifestParser::ParseRule(string* err) { - string name; - if (!lexer_.ReadIdent(&name)) - return lexer_.Error("expected rule name", err); - - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - if (state_->LookupRule(name) != NULL) { - *err = "duplicate rule '" + name + "'"; - return false; - } - - Rule* rule = new Rule(name); // XXX scoped_ptr - - while (lexer_.PeekToken(Lexer::INDENT)) { - string key; - EvalString value; - if (!ParseLet(&key, &value, err)) - return false; - - if (key == "command") { - rule->command_ = value; - } else if (key == "depfile") { - rule->depfile_ = value; - } else if (key == "description") { - rule->description_ = value; - } else if (key == "generator") { - rule->generator_ = true; - } else if (key == "restat") { - rule->restat_ = true; - } else if (key == "rspfile") { - rule->rspfile_ = value; - } else if (key == "rspfile_content") { - rule->rspfile_content_ = value; - } else { - // Die on other keyvals for now; revisit if we want to add a - // scope here. - return lexer_.Error("unexpected variable '" + key + "'", err); - } - } - - if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) - return lexer_.Error("rspfile and rspfile_content need to be both specified", err); - - if (rule->command_.empty()) - return lexer_.Error("expected 'command =' line", err); - - state_->AddRule(rule); - return true; -} - -bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) { - if (!lexer_.ReadIdent(key)) - return false; - if (!ExpectToken(Lexer::EQUALS, err)) - return false; - if (!lexer_.ReadVarValue(value, err)) - return false; - return true; -} - -bool ManifestParser::ParseDefault(string* err) { - EvalString eval; - if (!lexer_.ReadPath(&eval, err)) - return false; - if (eval.empty()) - return lexer_.Error("expected target name", err); - - do { - string path = eval.Evaluate(env_); - string path_err; - if (!CanonicalizePath(&path, &path_err)) - return lexer_.Error(path_err, err); - if (!state_->AddDefault(path, &path_err)) - return lexer_.Error(path_err, err); - - eval.Clear(); - if (!lexer_.ReadPath(&eval, err)) - return false; - } while (!eval.empty()); - - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - return true; -} - -bool ManifestParser::ParseEdge(string* err) { - vector ins, outs; - - { - EvalString out; - if (!lexer_.ReadPath(&out, err)) - return false; - if (out.empty()) - return lexer_.Error("expected path", err); - - do { - outs.push_back(out); - - out.Clear(); - if (!lexer_.ReadPath(&out, err)) - return false; - } while (!out.empty()); - } - - if (!ExpectToken(Lexer::COLON, err)) - return false; - - string rule_name; - if (!lexer_.ReadIdent(&rule_name)) - return lexer_.Error("expected build command name", err); - - const Rule* rule = state_->LookupRule(rule_name); - if (!rule) - return lexer_.Error("unknown build rule '" + rule_name + "'", err); - - for (;;) { - // XXX should we require one path here? - EvalString in; - if (!lexer_.ReadPath(&in, err)) - return false; - if (in.empty()) - break; - ins.push_back(in); - } - - // Add all implicit deps, counting how many as we go. - int implicit = 0; - if (lexer_.PeekToken(Lexer::PIPE)) { - for (;;) { - EvalString in; - if (!lexer_.ReadPath(&in, err)) - return err; - if (in.empty()) - break; - ins.push_back(in); - ++implicit; - } - } - - // Add all order-only deps, counting how many as we go. - int order_only = 0; - if (lexer_.PeekToken(Lexer::PIPE2)) { - for (;;) { - EvalString in; - if (!lexer_.ReadPath(&in, err)) - return false; - if (in.empty()) - break; - ins.push_back(in); - ++order_only; - } - } - - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - // Default to using outer env. - BindingEnv* env = env_; - - // But create and fill a nested env if there are variables in scope. - if (lexer_.PeekToken(Lexer::INDENT)) { - // XXX scoped_ptr to handle error case. - env = new BindingEnv(env_); - do { - string key; - EvalString val; - if (!ParseLet(&key, &val, err)) - return false; - env->AddBinding(key, val.Evaluate(env_)); - } while (lexer_.PeekToken(Lexer::INDENT)); - } - - Edge* edge = state_->AddEdge(rule); - edge->env_ = env; - for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { - string path = i->Evaluate(env); - string path_err; - if (!CanonicalizePath(&path, &path_err)) - return lexer_.Error(path_err, err); - state_->AddIn(edge, path); - } - for (vector::iterator i = outs.begin(); i != outs.end(); ++i) { - string path = i->Evaluate(env); - string path_err; - if (!CanonicalizePath(&path, &path_err)) - return lexer_.Error(path_err, err); - state_->AddOut(edge, path); - } - edge->implicit_deps_ = implicit; - edge->order_only_deps_ = order_only; - - return true; -} - -bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { - // XXX this should use ReadPath! - EvalString eval; - if (!lexer_.ReadPath(&eval, err)) - return false; - string path = eval.Evaluate(env_); - - string contents; - string read_err; - if (!file_reader_->ReadFile(path, &contents, &read_err)) - return lexer_.Error("loading '" + path + "': " + read_err, err); - - ManifestParser subparser(state_, file_reader_); - if (new_scope) { - subparser.env_ = new BindingEnv(env_); - } else { - subparser.env_ = env_; - } - - if (!subparser.Parse(path, contents, err)) - return false; - - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - return true; -} - -bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) { - Lexer::Token token = lexer_.ReadToken(); - if (token != expected) { - string message = string("expected ") + Lexer::TokenName(expected); - message += string(", got ") + Lexer::TokenName(token); - message += Lexer::TokenErrorHint(expected); - return lexer_.Error(message, err); - } - return true; -} diff --git a/src/parsers.h b/src/parsers.h deleted file mode 100644 index f889156..0000000 --- a/src/parsers.h +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef NINJA_PARSERS_H_ -#define NINJA_PARSERS_H_ - -#include -#include -#include - -using namespace std; - -#include "lexer.h" -#include "string_piece.h" - -struct BindingEnv; -struct EvalString; -struct State; - -/// Parses .ninja files. -struct ManifestParser { - struct FileReader { - virtual ~FileReader() {} - virtual bool ReadFile(const string& path, string* content, string* err) = 0; - }; - - ManifestParser(State* state, FileReader* file_reader); - - /// Load and parse a file. - bool Load(const string& filename, string* err); - - /// Parse a text string of input. Used by tests. - bool ParseTest(const string& input, string* err) { - return Parse("input", input, err); - } - -private: - /// Parse a file, given its contents as a string. - bool Parse(const string& filename, const string& input, string* err); - - /// Parse various statement types. - bool ParseRule(string* err); - bool ParseLet(string* key, EvalString* val, string* err); - bool ParseEdge(string* err); - bool ParseDefault(string* err); - - /// Parse either a 'subninja' or 'include' line. - bool ParseFileInclude(bool new_scope, string* err); - - /// If the next token is not \a expected, produce an error string - /// saying "expectd foo, got bar". - bool ExpectToken(Lexer::Token expected, string* err); - - State* state_; - BindingEnv* env_; - FileReader* file_reader_; - Lexer lexer_; -}; - -#endif // NINJA_PARSERS_H_ diff --git a/src/parsers_test.cc b/src/parsers_test.cc deleted file mode 100644 index fc83946..0000000 --- a/src/parsers_test.cc +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "parsers.h" - -#include - -#include "graph.h" -#include "state.h" - -struct ParserTest : public testing::Test, - public ManifestParser::FileReader { - void AssertParse(const char* input) { - ManifestParser parser(&state, this); - string err; - ASSERT_TRUE(parser.ParseTest(input, &err)) << err; - ASSERT_EQ("", err); - } - - virtual bool ReadFile(const string& path, string* content, string* err) { - files_read_.push_back(path); - map::iterator i = files_.find(path); - if (i == files_.end()) { - *err = "No such file or directory"; // Match strerror() for ENOENT. - return false; - } - *content = i->second; - return true; - } - - State state; - map files_; - vector files_read_; -}; - -TEST_F(ParserTest, Empty) { - ASSERT_NO_FATAL_FAILURE(AssertParse("")); -} - -TEST_F(ParserTest, Rules) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"\n" -"rule date\n" -" command = date > $out\n" -"\n" -"build result: cat in_1.cc in-2.O\n")); - - ASSERT_EQ(3u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; - EXPECT_EQ("cat", rule->name()); - EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize()); -} - -TEST_F(ParserTest, IgnoreIndentedComments) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -" #indented comment\n" -"rule cat\n" -" command = cat $in > $out\n" -" #generator = 1\n" -" restat = 1 # comment\n" -" #comment\n" -"build result: cat in_1.cc in-2.O\n" -" #comment\n")); - - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; - EXPECT_EQ("cat", rule->name()); - EXPECT_TRUE(rule->restat()); - EXPECT_FALSE(rule->generator()); -} - -TEST_F(ParserTest, IgnoreIndentedBlankLines) { - // the indented blanks used to cause parse errors - ASSERT_NO_FATAL_FAILURE(AssertParse( -" \n" -"rule cat\n" -" command = cat $in > $out\n" -" \n" -"build result: cat in_1.cc in-2.O\n" -" \n" -"variable=1\n")); - - // the variable must be in the top level environment - EXPECT_EQ("1", state.bindings_.LookupVariable("variable")); -} - -TEST_F(ParserTest, ResponseFiles) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat_rsp\n" -" command = cat $rspfile > $out\n" -" rspfile = $rspfile\n" -" rspfile_content = $in\n" -"\n" -"build out: cat_rsp in\n" -" rspfile=out.rsp\n")); - - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; - EXPECT_EQ("cat_rsp", rule->name()); - EXPECT_EQ("[cat ][$rspfile][ > ][$out]", rule->command().Serialize()); - EXPECT_EQ("[$rspfile]", rule->rspfile().Serialize()); - EXPECT_EQ("[$in]", rule->rspfile_content().Serialize()); -} - -TEST_F(ParserTest, InNewline) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat_rsp\n" -" command = cat $in_newline > $out\n" -"\n" -"build out: cat_rsp in in2\n" -" rspfile=out.rsp\n")); - - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; - EXPECT_EQ("cat_rsp", rule->name()); - EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->command().Serialize()); - - Edge* edge = state.edges_[0]; - EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand()); -} - -TEST_F(ParserTest, Variables) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"l = one-letter-test\n" -"rule link\n" -" command = ld $l $extra $with_under -o $out $in\n" -"\n" -"extra = -pthread\n" -"with_under = -under\n" -"build a: link b c\n" -"nested1 = 1\n" -"nested2 = $nested1/2\n" -"build supernested: link x\n" -" extra = $nested2/3\n")); - - ASSERT_EQ(2u, state.edges_.size()); - Edge* edge = state.edges_[0]; - EXPECT_EQ("ld one-letter-test -pthread -under -o a b c", - edge->EvaluateCommand()); - EXPECT_EQ("1/2", state.bindings_.LookupVariable("nested2")); - - edge = state.edges_[1]; - EXPECT_EQ("ld one-letter-test 1/2/3 -under -o supernested x", - edge->EvaluateCommand()); -} - -TEST_F(ParserTest, VariableScope) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"foo = bar\n" -"rule cmd\n" -" command = cmd $foo $in $out\n" -"\n" -"build inner: cmd a\n" -" foo = baz\n" -"build outer: cmd b\n" -"\n" // Extra newline after build line tickles a regression. -)); - - ASSERT_EQ(2u, state.edges_.size()); - EXPECT_EQ("cmd baz a inner", state.edges_[0]->EvaluateCommand()); - EXPECT_EQ("cmd bar b outer", state.edges_[1]->EvaluateCommand()); -} - -TEST_F(ParserTest, Continuation) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule link\n" -" command = foo bar $\n" -" baz\n" -"\n" -"build a: link c $\n" -" d e f\n")); - - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; - EXPECT_EQ("link", rule->name()); - EXPECT_EQ("[foo bar baz]", rule->command().Serialize()); -} - -TEST_F(ParserTest, Backslash) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"foo = bar\\baz\n" -"foo2 = bar\\ baz\n" -)); - EXPECT_EQ("bar\\baz", state.bindings_.LookupVariable("foo")); - EXPECT_EQ("bar\\ baz", state.bindings_.LookupVariable("foo2")); -} - -TEST_F(ParserTest, Comment) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"# this is a comment\n" -"foo = not # a comment\n")); - EXPECT_EQ("not # a comment", state.bindings_.LookupVariable("foo")); -} - -TEST_F(ParserTest, Dollars) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule foo\n" -" command = ${out}bar$$baz$$$\n" -"blah\n" -"x = $$dollar\n" -"build $x: foo y\n" -)); - EXPECT_EQ("$dollar", state.bindings_.LookupVariable("x")); - EXPECT_EQ("$dollarbar$baz$blah", state.edges_[0]->EvaluateCommand()); -} - -TEST_F(ParserTest, EscapeSpaces) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule spaces\n" -" command = something\n" -"build foo$ bar: spaces $$one two$$$ three\n" -)); - EXPECT_TRUE(state.LookupNode("foo bar")); - EXPECT_EQ(state.edges_[0]->outputs_[0]->path(), "foo bar"); - EXPECT_EQ(state.edges_[0]->inputs_[0]->path(), "$one"); - EXPECT_EQ(state.edges_[0]->inputs_[1]->path(), "two$ three"); - EXPECT_EQ(state.edges_[0]->EvaluateCommand(), "something"); -} - -TEST_F(ParserTest, CanonicalizeFile) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"build out: cat in/1 in//2\n" -"build in/1: cat\n" -"build in/2: cat\n")); - - EXPECT_TRUE(state.LookupNode("in/1")); - EXPECT_TRUE(state.LookupNode("in/2")); - EXPECT_FALSE(state.LookupNode("in//1")); - EXPECT_FALSE(state.LookupNode("in//2")); -} - -TEST_F(ParserTest, PathVariables) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"dir = out\n" -"build $dir/exe: cat src\n")); - - EXPECT_FALSE(state.LookupNode("$dir/exe")); - EXPECT_TRUE(state.LookupNode("out/exe")); -} - -TEST_F(ParserTest, CanonicalizePaths) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"build ./out.o: cat ./bar/baz/../foo.cc\n")); - - EXPECT_FALSE(state.LookupNode("./out.o")); - EXPECT_TRUE(state.LookupNode("out.o")); - EXPECT_FALSE(state.LookupNode("./bar/baz/../foo.cc")); - EXPECT_TRUE(state.LookupNode("bar/foo.cc")); -} - -TEST_F(ParserTest, ReservedWords) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule build\n" -" command = rule run $out\n" -"build subninja: build include default foo.cc\n" -"default subninja\n")); -} - -TEST_F(ParserTest, Errors) { - { - ManifestParser parser(NULL, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("foobar", &err)); - EXPECT_EQ("input:1: expected '=', got eof\n" - "foobar\n" - " ^ near here" - , err); - } - - { - ManifestParser parser(NULL, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x 3", &err)); - EXPECT_EQ("input:1: expected '=', got identifier\n" - "x 3\n" - " ^ near here" - , err); - } - - { - ManifestParser parser(NULL, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x = 3", &err)); - EXPECT_EQ("input:1: unexpected EOF\n" - "x = 3\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x = 3\ny 2", &err)); - EXPECT_EQ("input:2: expected '=', got identifier\n" - "y 2\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x = $", &err)); - EXPECT_EQ("input:1: bad $-escape (literal $ must be written as $$)\n" - "x = $\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x = $\n $[\n", &err)); - EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n" - " $[\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("x = a$\n b$\n $\n", &err)); - EXPECT_EQ("input:4: unexpected EOF\n" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("build x: y z\n", &err)); - EXPECT_EQ("input:1: unknown build rule 'y'\n" - "build x: y z\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("build x:: y z\n", &err)); - EXPECT_EQ("input:1: expected build command name\n" - "build x:: y z\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n command = cat ok\n" - "build x: cat $\n :\n", - &err)); - EXPECT_EQ("input:4: expected newline, got ':'\n" - " :\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n", - &err)); - EXPECT_EQ("input:2: expected 'command =' line\n", err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n" - " command = ${fafsd\n" - "foo = bar\n", - &err)); - EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n" - " command = ${fafsd\n" - " ^ near here" - , err); - } - - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n" - " command = cat\n" - "build $.: cat foo\n", - &err)); - EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n" - "build $.: cat foo\n" - " ^ near here" - , err); - } - - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n" - " command = cat\n" - "build $: cat foo\n", - &err)); - EXPECT_EQ("input:3: expected ':', got newline ($ also escapes ':')\n" - "build $: cat foo\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule %foo\n", - &err)); - EXPECT_EQ("input:1: expected rule name\n", err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cc\n" - " command = foo\n" - " othervar = bar\n", - &err)); - EXPECT_EQ("input:3: unexpected variable 'othervar'\n" - " othervar = bar\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" - "build $.: cc bar.cc\n", - &err)); - EXPECT_EQ("input:3: bad $-escape (literal $ must be written as $$)\n" - "build $.: cc bar.cc\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" - "build $: cc bar.cc\n", - &err)); - EXPECT_EQ("input:3: expected ':', got newline ($ also escapes ':')\n" - "build $: cc bar.cc\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("default\n", - &err)); - EXPECT_EQ("input:1: expected target name\n" - "default\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("default nonexistent\n", - &err)); - EXPECT_EQ("input:1: unknown target 'nonexistent'\n" - "default nonexistent\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule r\n command = r\n" - "build b: r\n" - "default b:\n", - &err)); - EXPECT_EQ("input:4: expected newline, got ':'\n" - "default b:\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("default $a\n", &err)); - EXPECT_EQ("input:1: empty path\n" - "default $a\n" - " ^ near here" - , err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_FALSE(parser.ParseTest("rule r\n" - " command = r\n" - "build $a: r $c\n", &err)); - // XXX the line number is wrong; we should evaluate paths in ParseEdge - // as we see them, not after we've read them all! - EXPECT_EQ("input:4: empty path\n", err); - } - - { - State state; - ManifestParser parser(&state, NULL); - string err; - // the indented blank line must terminate the rule - // this also verifies that "unexpected (token)" errors are correct - EXPECT_FALSE(parser.ParseTest("rule r\n" - " command = r\n" - " \n" - " generator = 1\n", &err)); - EXPECT_EQ("input:4: unexpected indent\n", err); - } -} - -TEST_F(ParserTest, MissingInput) { - State state; - ManifestParser parser(&state, this); - string err; - EXPECT_FALSE(parser.Load("build.ninja", &err)); - EXPECT_EQ("loading 'build.ninja': No such file or directory", err); -} - -TEST_F(ParserTest, MultipleOutputs) { - State state; - ManifestParser parser(&state, NULL); - string err; - EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n depfile = bar\n" - "build a.o b.o: cc c.cc\n", - &err)); - EXPECT_EQ("", err); -} - -TEST_F(ParserTest, SubNinja) { - files_["test.ninja"] = - "var = inner\n" - "build $builddir/inner: varref\n"; - ASSERT_NO_FATAL_FAILURE(AssertParse( -"builddir = some_dir/\n" -"rule varref\n" -" command = varref $var\n" -"var = outer\n" -"build $builddir/outer: varref\n" -"subninja test.ninja\n" -"build $builddir/outer2: varref\n")); - ASSERT_EQ(1u, files_read_.size()); - - EXPECT_EQ("test.ninja", files_read_[0]); - EXPECT_TRUE(state.LookupNode("some_dir/outer")); - // Verify our builddir setting is inherited. - EXPECT_TRUE(state.LookupNode("some_dir/inner")); - - ASSERT_EQ(3u, state.edges_.size()); - EXPECT_EQ("varref outer", state.edges_[0]->EvaluateCommand()); - EXPECT_EQ("varref inner", state.edges_[1]->EvaluateCommand()); - EXPECT_EQ("varref outer", state.edges_[2]->EvaluateCommand()); -} - -TEST_F(ParserTest, MissingSubNinja) { - ManifestParser parser(&state, this); - string err; - EXPECT_FALSE(parser.ParseTest("subninja foo.ninja\n", &err)); - EXPECT_EQ("input:1: loading 'foo.ninja': No such file or directory\n" - "subninja foo.ninja\n" - " ^ near here" - , err); -} - -TEST_F(ParserTest, Include) { - files_["include.ninja"] = "var = inner\n"; - ASSERT_NO_FATAL_FAILURE(AssertParse( -"var = outer\n" -"include include.ninja\n")); - - ASSERT_EQ(1u, files_read_.size()); - EXPECT_EQ("include.ninja", files_read_[0]); - EXPECT_EQ("inner", state.bindings_.LookupVariable("var")); -} - -TEST_F(ParserTest, Implicit) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"build foo: cat bar | baz\n")); - - Edge* edge = state.LookupNode("foo")->in_edge(); - ASSERT_TRUE(edge->is_implicit(1)); -} - -TEST_F(ParserTest, OrderOnly) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n command = cat $in > $out\n" -"build foo: cat bar || baz\n")); - - Edge* edge = state.LookupNode("foo")->in_edge(); - ASSERT_TRUE(edge->is_order_only(1)); -} - -TEST_F(ParserTest, DefaultDefault) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n command = cat $in > $out\n" -"build a: cat foo\n" -"build b: cat foo\n" -"build c: cat foo\n" -"build d: cat foo\n")); - - string err; - EXPECT_EQ(4u, state.DefaultNodes(&err).size()); - EXPECT_EQ("", err); -} - -TEST_F(ParserTest, DefaultStatements) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n command = cat $in > $out\n" -"build a: cat foo\n" -"build b: cat foo\n" -"build c: cat foo\n" -"build d: cat foo\n" -"third = c\n" -"default a b\n" -"default $third\n")); - - string err; - std::vector nodes = state.DefaultNodes(&err); - EXPECT_EQ("", err); - ASSERT_EQ(3u, nodes.size()); - EXPECT_EQ("a", nodes[0]->path()); - EXPECT_EQ("b", nodes[1]->path()); - EXPECT_EQ("c", nodes[2]->path()); -} - -TEST_F(ParserTest, UTF8) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule utf8\n" -" command = true\n" -" description = compilaci\xC3\xB3\n")); -} diff --git a/src/test.cc b/src/test.cc index afedd10..0138b3a 100644 --- a/src/test.cc +++ b/src/test.cc @@ -19,7 +19,7 @@ #include #include "build_log.h" -#include "parsers.h" +#include "manifest_parser.h" #include "util.h" #ifdef _WIN32 -- cgit v0.12