diff options
| author | Evan Martin <martine@danga.com> | 2010-12-17 23:21:13 (GMT) |
|---|---|---|
| committer | Evan Martin <martine@danga.com> | 2010-12-17 23:21:13 (GMT) |
| commit | 3261f368ba73ece86d0fb190bc6205ca682e1ae3 (patch) | |
| tree | 89861c9286d55064b34fda723fc02471cf8f69ca | |
| parent | b0f931f18d4ce98ebbc712d0caa8368b68137028 (diff) | |
| download | Ninja-3261f368ba73ece86d0fb190bc6205ca682e1ae3.zip Ninja-3261f368ba73ece86d0fb190bc6205ca682e1ae3.tar.gz Ninja-3261f368ba73ece86d0fb190bc6205ca682e1ae3.tar.bz2 | |
simplify token representation, speeding up parse
| -rw-r--r-- | src/parsers.cc | 17 | ||||
| -rw-r--r-- | src/parsers.h | 4 |
2 files changed, 11 insertions, 10 deletions
diff --git a/src/parsers.cc b/src/parsers.cc index 4d6c058..4041bf6 100644 --- a/src/parsers.cc +++ b/src/parsers.cc @@ -9,8 +9,8 @@ string Token::AsString() const { switch (type_) { - case IDENT: return "'" + extra_ + "'"; - case UNKNOWN: return "unknown '" + extra_ + "'"; + case IDENT: return "'" + string(pos_, end_ - pos_) + "'"; + case UNKNOWN: return "unknown '" + string(pos_, end_ - pos_) + "'"; case RULE: return "'rule'"; case BUILD: return "'build'"; case SUBNINJA: return "'subninja'"; @@ -99,7 +99,7 @@ bool Tokenizer::ReadIdent(string* out) { PeekToken(); if (token_.type_ != Token::IDENT) return false; - out->assign(token_.extra_); + out->assign(token_.pos_, token_.end_ - token_.pos_); ConsumeToken(); return true; } @@ -159,14 +159,15 @@ Token::Type Tokenizer::PeekToken() { if (IsIdentChar(*cur_)) { while (cur_ < end_ && IsIdentChar(*cur_)) { - token_.extra_.push_back(*cur_); ++cur_; } - if (token_.extra_ == "rule") + token_.end_ = cur_; + int len = token_.end_ - token_.pos_; + if (len == 4 && memcmp(token_.pos_, "rule", 4) == 0) token_.type_ = Token::RULE; - else if (token_.extra_ == "build") + else if (len == 5 && memcmp(token_.pos_, "build", 5) == 0) token_.type_ = Token::BUILD; - else if (token_.extra_ == "subninja") + else if (len == 8 && memcmp(token_.pos_, "subninja", 8) == 0) token_.type_ = Token::SUBNINJA; else token_.type_ = Token::IDENT; @@ -191,7 +192,7 @@ Token::Type Tokenizer::PeekToken() { if (token_.type_ == Token::NONE) { token_.type_ = Token::UNKNOWN; - token_.extra_ = *cur_; + token_.end_ = cur_ + 1; } return token_.type_; diff --git a/src/parsers.h b/src/parsers.h index fc20436..a7f23e5 100644 --- a/src/parsers.h +++ b/src/parsers.h @@ -26,12 +26,12 @@ struct Token { }; explicit Token(Type type) : type_(type) {} - void Clear() { type_ = NONE; extra_.clear(); } + void Clear() { type_ = NONE; } string AsString() const; Type type_; const char* pos_; - string extra_; + const char* end_; }; struct Tokenizer { |