switch the core ninja parser to use re2c for the lexer

- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
  jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.

1 files changed, 85 insertions, 0 deletions

diff --git a/src/lexer_test.cc b/src/lexer_test.cc
new file mode 100644
index 0000000..ce8082a
--- /dev/null
+++ b/src/lexer_test.cc
@@ -0,0 +1,85 @@
+// Copyright 2011 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "lexer.h"
+
+#include <gtest/gtest.h>
+
+#include "eval_env.h"
+
+TEST(Lexer, ReadVarValue) {
+  Lexer lexer("plain text $var $VaR ${x}\n");
+  EvalString eval;
+  string err;
+  EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+  EXPECT_EQ("", err);
+  EXPECT_EQ("[plain text ][$var][ ][$VaR][ ][$x]",
+            eval.Serialize());
+}
+
+TEST(Lexer, ReadEvalStringEscapes) {
+  Lexer lexer("$ $$ab $\ncde\n");
+  EvalString eval;
+  string err;
+  EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+  EXPECT_EQ("", err);
+  EXPECT_EQ("[ $ab cde]",
+            eval.Serialize());
+}
+
+TEST(Lexer, ReadIdent) {
+  Lexer lexer("foo baR baz_123 blah.dots");
+  string ident;
+  EXPECT_TRUE(lexer.ReadIdent(&ident));
+  EXPECT_EQ("foo", ident);
+  EXPECT_TRUE(lexer.ReadIdent(&ident));
+  EXPECT_EQ("baR", ident);
+  EXPECT_TRUE(lexer.ReadIdent(&ident));
+  EXPECT_EQ("baz_123", ident);
+}
+
+TEST(Lexer, ReadIdentCurlies) {
+  // Verify that ReadIdent includes dots in the name,
+  // but in an expansion $bar.dots stops at the dot.
+  Lexer lexer("foo.dots $bar.dots ${bar.dots}\n");
+  string ident;
+  EXPECT_TRUE(lexer.ReadIdent(&ident));
+  EXPECT_EQ("foo.dots", ident);
+
+  EvalString eval;
+  string err;
+  EXPECT_TRUE(lexer.ReadVarValue(&eval, &err));
+  EXPECT_EQ("", err);
+  EXPECT_EQ("[$bar][.dots ][$bar.dots]",
+            eval.Serialize());
+}
+
+TEST(Lexer, Error) {
+  Lexer lexer("foo$\nbad $");
+  EvalString eval;
+  string err;
+  ASSERT_FALSE(lexer.ReadVarValue(&eval, &err));
+  EXPECT_EQ("input:2: bad $-escape (literal $ must be written as $$)\n"
+            "bad $\n"
+            "    ^ near here\n"
+            , err);
+}
+
+TEST(Lexer, CommentEOF) {
+  // Verify we don't run off the end of the string when the EOF is
+  // mid-comment.
+  Lexer lexer("# foo");
+  Lexer::Token token = lexer.ReadToken();
+  EXPECT_EQ(Lexer::ERROR, token);
+}