switch the core ninja parser to use re2c for the lexer

- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
  jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.

1 files changed, 10 insertions, 4 deletions

diff --git a/src/eval_env.h b/src/eval_env.h
index ed7c2f4..8c144f0 100644
--- a/src/eval_env.h
+++ b/src/eval_env.h
@@ -20,6 +20,8 @@
 #include <vector>
 using namespace std;
 
+#include "string_piece.h"
+
 /// An interface for a scope for variable (e.g. "$foo") lookups.
 struct Env {
   virtual ~Env() {}
@@ -41,14 +43,18 @@ struct BindingEnv : public Env {
 /// A tokenized string that contains variable references.
 /// Can be evaluated relative to an Env.
 struct EvalString {
-  bool Parse(const string& input, string* err, size_t* err_index=NULL);
   string Evaluate(Env* env) const;
 
-  const string& unparsed() const { return unparsed_; }
-  bool empty() const { return unparsed_.empty(); }
+  void Clear() { parsed_.clear(); }
+  bool empty() const { return parsed_.empty(); }
 
-  string unparsed_;
   enum TokenType { RAW, SPECIAL };
+  void Add(TokenType type, StringPiece text);
+
+  /// Construct a human-readable representation of the parsed state,
+  /// for use in tests.
+  string Serialize() const;
+
   typedef vector<pair<string, TokenType> > TokenList;
   TokenList parsed_;
 };