diff options
author | Evan Martin <martine@danga.com> | 2011-12-29 21:00:27 (GMT) |
---|---|---|
committer | Evan Martin <martine@danga.com> | 2011-12-29 21:14:39 (GMT) |
commit | 8a0c96075786c1983bdfa2f37f32b75200ea0334 (patch) | |
tree | 95e2b0c24aedcda9ec5ed09329e69fd7a1925212 /src/parsers.h | |
parent | ad7d9f43f1bd8e04321d8fdb07ebf7b96ab525a1 (diff) | |
download | Ninja-8a0c96075786c1983bdfa2f37f32b75200ea0334.zip Ninja-8a0c96075786c1983bdfa2f37f32b75200ea0334.tar.gz Ninja-8a0c96075786c1983bdfa2f37f32b75200ea0334.tar.bz2 |
switch the core ninja parser to use re2c for the lexer
- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.
Diffstat (limited to 'src/parsers.h')
-rw-r--r-- | src/parsers.h | 97 |
1 files changed, 19 insertions, 78 deletions
diff --git a/src/parsers.h b/src/parsers.h index 101b278..f889156 100644 --- a/src/parsers.h +++ b/src/parsers.h @@ -21,74 +21,10 @@ using namespace std; +#include "lexer.h" #include "string_piece.h" struct BindingEnv; - -/// A single parsed token in an input stream. -struct Token { - enum Type { - NONE, - UNKNOWN, - IDENT, - NEWLINE, - EQUALS, - COLON, - PIPE, - PIPE2, - INDENT, - OUTDENT, - TEOF - }; - explicit Token(Type type) : type_(type) {} - - void Clear() { type_ = NONE; } - string AsString() const; - - Type type_; - const char* pos_; - const char* end_; -}; - -/// Processes an input stream into Tokens. -struct Tokenizer { - Tokenizer() : - token_(Token::NONE), - last_indent_(0), cur_indent_(-1) {} - - void Start(const char* start, const char* end); - /// Report an error at a particular location. - bool ErrorAt(const char* pos, const string& message, string* err); - /// Report an error with a location pointing at the current token. - bool Error(const string& message, string* err) { - return ErrorAt(token_.pos_, message, err); - } - /// Call Error() with "expected foo, got bar". - bool ErrorExpected(const string& expected, string* err); - - const Token& token() const { return token_; } - - void SkipWhitespace(bool newline=false); - bool Newline(string* err); - bool ExpectToken(Token::Type expected, string* err); - bool ExpectIdent(const char* expected, string* err); - bool ReadIdent(StringPiece* out); - bool ReadIdent(string* out); - bool ReadToNewline(string* text, string* err, - size_t max_length=std::numeric_limits<size_t>::max()); - - Token::Type PeekToken(); - void ConsumeToken(); - - const char* start_; /// Start of the input. - const char* cur_; /// Current position within the input. - const char* end_; /// End of the input. - - const char* cur_line_; /// Start of current line. - Token token_; - int last_indent_, cur_indent_; -}; - struct EvalString; struct State; @@ -101,30 +37,35 @@ struct ManifestParser { ManifestParser(State* state, FileReader* file_reader); + /// Load and parse a file. bool Load(const string& filename, string* err); - bool Parse(const string& input, string* err); + /// Parse a text string of input. Used by tests. + bool ParseTest(const string& input, string* err) { + return Parse("input", input, err); + } + +private: + /// Parse a file, given its contents as a string. + bool Parse(const string& filename, const string& input, string* err); + + /// Parse various statement types. bool ParseRule(string* err); - /// Parse a key=val statement, expanding $vars in the value with the - /// current env. - bool ParseLet(string* key, string* val, string* err); + bool ParseLet(string* key, EvalString* val, string* err); bool ParseEdge(string* err); - bool ParseDefaults(string* err); + bool ParseDefault(string* err); /// Parse either a 'subninja' or 'include' line. - bool ParseFileInclude(string* err); - + bool ParseFileInclude(bool new_scope, string* err); - /// Parse the "key=" half of a key=val statement. - bool ParseLetKey(string* key, string* err); - /// Parse the val half of a key=val statement, writing and parsing - /// output into an EvalString (ready for expansion). - bool ParseLetValue(EvalString* eval, string* err); + /// If the next token is not \a expected, produce an error string + /// saying "expectd foo, got bar". + bool ExpectToken(Lexer::Token expected, string* err); State* state_; BindingEnv* env_; FileReader* file_reader_; - Tokenizer tokenizer_; + Lexer lexer_; }; #endif // NINJA_PARSERS_H_ |