switch the core ninja parser to use re2c for the lexer

- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
  jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.

1 files changed, 6 insertions, 3 deletions

diff --git a/src/state_test.cc b/src/state_test.cc
index b9e55cb..ca4e60c 100644
--- a/src/state_test.cc
+++ b/src/state_test.cc
@@ -21,11 +21,14 @@ namespace {
 
 TEST(State, Basic) {
   State state;
+
   Rule* rule = new Rule("cat");
-  string err;
-  EXPECT_TRUE(rule->command().Parse("cat $in > $out", &err));
-  ASSERT_EQ("", err);
+  rule->command_.Add(EvalString::RAW, "cat ");
+  rule->command_.Add(EvalString::SPECIAL, "in");
+  rule->command_.Add(EvalString::RAW, " > ");
+  rule->command_.Add(EvalString::SPECIAL, "out");
   state.AddRule(rule);
+
   Edge* edge = state.AddEdge(rule);
   state.AddIn(edge, "in1");
   state.AddIn(edge, "in2");