From 65151e7eefce514febd0a9676c5363c6d4c54b66 Mon Sep 17 00:00:00 2001 From: Scott Graham Date: Thu, 30 Oct 2014 15:34:25 -0700 Subject: CanonicalizePath handles \ on Windows --- doc/manual.asciidoc | 4 +- misc/write_fake_manifests.py | 2 +- src/build_test.cc | 6 +- src/includes_normalize-win32.cc | 20 +- src/includes_normalize_test.cc | 30 +-- src/lexer.cc | 493 ++++++++++++++++++++++------------------ src/lexer.in.cc | 25 +- src/manifest_parser_test.cc | 22 +- src/util.cc | 40 +++- src/util_test.cc | 66 ++++++ 10 files changed, 431 insertions(+), 277 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index fcf3db3..3757c86 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -696,9 +696,7 @@ Lexical syntax Ninja is mostly encoding agnostic, as long as the bytes Ninja cares about (like slashes in paths) are ASCII. This means e.g. UTF-8 or -ISO-8859-1 input files ought to work. (To simplify some code, tabs -and carriage returns are currently disallowed; this could be fixed if -it really mattered to you.) +ISO-8859-1 input files ought to work. Comments begin with `#` and extend to the end of the line. diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py index 837007e..ca49535 100644 --- a/misc/write_fake_manifests.py +++ b/misc/write_fake_manifests.py @@ -182,7 +182,7 @@ def FileWriter(path): def random_targets(): - num_targets = 800 + num_targets = 1500 gen = GenRandom() # N-1 static libraries, and 1 executable depending on all of them. diff --git a/src/build_test.cc b/src/build_test.cc index 6336206..a5b0972 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -755,7 +755,7 @@ TEST_F(BuildTest, MakeDirs) { #ifdef _WIN32 ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n")); - EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err)); + EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); #else ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n")); @@ -767,11 +767,7 @@ TEST_F(BuildTest, MakeDirs) { ASSERT_EQ("", err); ASSERT_EQ(2u, fs_.directories_made_.size()); EXPECT_EQ("subdir", fs_.directories_made_[0]); -#ifdef _WIN32 - EXPECT_EQ("subdir\\dir2", fs_.directories_made_[1]); -#else EXPECT_EQ("subdir/dir2", fs_.directories_made_[1]); -#endif } TEST_F(BuildTest, DepFileMissing) { diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 05ce75d..0d230f8 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -68,12 +68,15 @@ string IncludesNormalize::ToLower(const string& s) { string IncludesNormalize::AbsPath(StringPiece s) { char result[_MAX_PATH]; GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + for (char* c = result; *c; ++c) + if (*c == '\\') + *c = '/'; return result; } string IncludesNormalize::Relativize(StringPiece path, const string& start) { - vector start_list = Split(AbsPath(start), '\\'); - vector path_list = Split(AbsPath(path), '\\'); + vector start_list = Split(AbsPath(start), '/'); + vector path_list = Split(AbsPath(path), '/'); int i; for (i = 0; i < static_cast(min(start_list.size(), path_list.size())); ++i) { @@ -88,7 +91,7 @@ string IncludesNormalize::Relativize(StringPiece path, const string& start) { rel_list.push_back(path_list[j]); if (rel_list.size() == 0) return "."; - return Join(rel_list, '\\'); + return Join(rel_list, '/'); } string IncludesNormalize::Normalize(const string& input, @@ -96,19 +99,16 @@ string IncludesNormalize::Normalize(const string& input, char copy[_MAX_PATH]; size_t len = input.size(); strncpy(copy, input.c_str(), input.size() + 1); - for (size_t j = 0; j < len; ++j) - if (copy[j] == '/') - copy[j] = '\\'; string err; - if (!CanonicalizePath(copy, &len, &err)) { - Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); - } + if (!CanonicalizePath(copy, &len, &err)) + Warning("couldn't canonicalize '%s': %s\n", input.c_str(), err.c_str()); + StringPiece partially_fixed(copy, len); + string curdir; if (!relative_to) { curdir = AbsPath("."); relative_to = curdir.c_str(); } - StringPiece partially_fixed(copy, len); if (!SameDrive(partially_fixed, relative_to)) return partially_fixed.AsString(); return Relativize(partially_fixed, relative_to); diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index d9d21bf..c4c2476 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -22,8 +22,8 @@ TEST(IncludesNormalize, Simple) { EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\.\\b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\./b", NULL)); } namespace { @@ -42,21 +42,21 @@ TEST(IncludesNormalize, WithRelative) { EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); - EXPECT_EQ(string("..\\") + currentdir + string("\\a"), + EXPECT_EQ(string("../") + currentdir + string("/a"), IncludesNormalize::Normalize("a", "../b")); - EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), + EXPECT_EQ(string("../") + currentdir + string("/a/b"), IncludesNormalize::Normalize("a/b", "../c")); - EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); + EXPECT_EQ("../../a", IncludesNormalize::Normalize("a", "b/c")); EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); } TEST(IncludesNormalize, Case) { EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); EXPECT_EQ("BdEf", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); - EXPECT_EQ("A\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\.\\B", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\./B", NULL)); + EXPECT_EQ("A/b", IncludesNormalize::Normalize("A\\.\\b", NULL)); + EXPECT_EQ("a/b", IncludesNormalize::Normalize("a\\./b", NULL)); + EXPECT_EQ("A/B", IncludesNormalize::Normalize("A\\.\\B", NULL)); + EXPECT_EQ("A/B", IncludesNormalize::Normalize("A\\./B", NULL)); } TEST(IncludesNormalize, Join) { @@ -92,13 +92,13 @@ TEST(IncludesNormalize, DifferentDrive) { IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); EXPECT_EQ("stuff.h", IncludesNormalize::Normalize("P:\\Vs08\\stuff.h", "p:\\vs08")); - EXPECT_EQ("p:\\vs08\\stuff.h", + EXPECT_EQ("p:/vs08/stuff.h", IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "c:\\vs08")); - EXPECT_EQ("P:\\vs08\\stufF.h", + EXPECT_EQ("P:/vs08/stufF.h", IncludesNormalize::Normalize("P:\\vs08\\stufF.h", "D:\\stuff/things")); - EXPECT_EQ("P:\\vs08\\stuff.h", + EXPECT_EQ("P:/vs08/stuff.h", IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); - // TODO: this fails; fix it. - //EXPECT_EQ("P:\\wee\\stuff.h", - // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("P:/wee/stuff.h", + IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", + "D:\\stuff/things")); } diff --git a/src/lexer.cc b/src/lexer.cc index 685fe81..37b8678 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -103,8 +103,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -129,7 +127,7 @@ Lexer::Token Lexer::ReadToken() { unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 0, 64, 64, + 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -163,56 +161,66 @@ Lexer::Token Lexer::ReadToken() { }; yych = *p; - if (yych <= '^') { - if (yych <= ',') { - if (yych <= 0x1F) { - if (yych <= 0x00) goto yy22; - if (yych == '\n') goto yy6; - goto yy24; + if (yych <= 'Z') { + if (yych <= '#') { + if (yych <= '\f') { + if (yych <= 0x00) goto yy23; + if (yych == '\n') goto yy7; + goto yy25; } else { - if (yych <= ' ') goto yy2; - if (yych == '#') goto yy4; - goto yy24; + if (yych <= 0x1F) { + if (yych <= '\r') goto yy6; + goto yy25; + } else { + if (yych <= ' ') goto yy2; + if (yych <= '"') goto yy25; + goto yy4; + } } } else { - if (yych <= ':') { - if (yych == '/') goto yy24; - if (yych <= '9') goto yy21; - goto yy15; + if (yych <= '9') { + if (yych <= ',') goto yy25; + if (yych == '/') goto yy25; + goto yy22; } else { - if (yych <= '=') { - if (yych <= '<') goto yy24; - goto yy13; + if (yych <= '<') { + if (yych <= ':') goto yy16; + goto yy25; } else { - if (yych <= '@') goto yy24; - if (yych <= 'Z') goto yy21; - goto yy24; + if (yych <= '=') goto yy14; + if (yych <= '@') goto yy25; + goto yy22; } } } } else { if (yych <= 'i') { - if (yych <= 'b') { - if (yych == '`') goto yy24; - if (yych <= 'a') goto yy21; - goto yy8; + if (yych <= 'a') { + if (yych == '_') goto yy22; + if (yych <= '`') goto yy25; + goto yy22; } else { - if (yych == 'd') goto yy12; - if (yych <= 'h') goto yy21; - goto yy19; + if (yych <= 'c') { + if (yych <= 'b') goto yy9; + goto yy22; + } else { + if (yych <= 'd') goto yy13; + if (yych <= 'h') goto yy22; + goto yy20; + } } } else { if (yych <= 'r') { - if (yych == 'p') goto yy10; - if (yych <= 'q') goto yy21; - goto yy11; + if (yych == 'p') goto yy11; + if (yych <= 'q') goto yy22; + goto yy12; } else { if (yych <= 'z') { - if (yych <= 's') goto yy20; - goto yy21; + if (yych <= 's') goto yy21; + goto yy22; } else { - if (yych == '|') goto yy17; - goto yy24; + if (yych == '|') goto yy18; + goto yy25; } } } @@ -220,192 +228,203 @@ Lexer::Token Lexer::ReadToken() { yy2: yyaccept = 0; yych = *(q = ++p); - goto yy70; + goto yy73; yy3: { token = INDENT; break; } yy4: yyaccept = 1; yych = *(q = ++p); - if (yych <= 0x00) goto yy5; - if (yych != '\r') goto yy65; + if (yych >= 0x01) goto yy68; yy5: { token = ERROR; break; } yy6: - ++p; + yych = *++p; + if (yych == '\n') goto yy65; + goto yy5; yy7: - { token = NEWLINE; break; } -yy8: ++p; - if ((yych = *p) == 'u') goto yy59; - goto yy26; +yy8: + { token = NEWLINE; break; } yy9: - { token = IDENT; break; } + ++p; + if ((yych = *p) == 'u') goto yy60; + goto yy27; yy10: - yych = *++p; - if (yych == 'o') goto yy55; - goto yy26; + { token = IDENT; break; } yy11: yych = *++p; - if (yych == 'u') goto yy51; - goto yy26; + if (yych == 'o') goto yy56; + goto yy27; yy12: yych = *++p; - if (yych == 'e') goto yy44; - goto yy26; + if (yych == 'u') goto yy52; + goto yy27; yy13: + yych = *++p; + if (yych == 'e') goto yy45; + goto yy27; +yy14: ++p; { token = EQUALS; break; } -yy15: +yy16: ++p; { token = COLON; break; } -yy17: +yy18: ++p; - if ((yych = *p) == '|') goto yy42; + if ((yych = *p) == '|') goto yy43; { token = PIPE; break; } -yy19: - yych = *++p; - if (yych == 'n') goto yy35; - goto yy26; yy20: yych = *++p; - if (yych == 'u') goto yy27; - goto yy26; + if (yych == 'n') goto yy36; + goto yy27; yy21: yych = *++p; - goto yy26; + if (yych == 'u') goto yy28; + goto yy27; yy22: + yych = *++p; + goto yy27; +yy23: ++p; { token = TEOF; break; } -yy24: +yy25: yych = *++p; goto yy5; -yy25: +yy26: ++p; yych = *p; -yy26: +yy27: if (yybm[0+yych] & 32) { - goto yy25; + goto yy26; } - goto yy9; -yy27: + goto yy10; +yy28: yych = *++p; - if (yych != 'b') goto yy26; + if (yych != 'b') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'j') goto yy26; + if (yych != 'j') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = SUBNINJA; break; } -yy35: +yy36: yych = *++p; - if (yych != 'c') goto yy26; + if (yych != 'c') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = INCLUDE; break; } -yy42: +yy43: ++p; { token = PIPE2; break; } -yy44: +yy45: yych = *++p; - if (yych != 'f') goto yy26; + if (yych != 'f') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 't') goto yy26; + if (yych != 't') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = DEFAULT; break; } -yy51: +yy52: yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = RULE; break; } -yy55: +yy56: yych = *++p; - if (yych != 'o') goto yy26; + if (yych != 'o') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = POOL; break; } -yy59: +yy60: yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = BUILD; break; } -yy64: +yy65: + ++p; + { token = NEWLINE; break; } +yy67: ++p; yych = *p; -yy65: +yy68: if (yybm[0+yych] & 64) { - goto yy64; + goto yy67; } - if (yych <= 0x00) goto yy66; - if (yych <= '\f') goto yy67; -yy66: + if (yych >= 0x01) goto yy70; +yy69: p = q; if (yyaccept <= 0) { goto yy3; } else { goto yy5; } -yy67: +yy70: ++p; { continue; } -yy69: +yy72: yyaccept = 0; q = ++p; yych = *p; -yy70: +yy73: if (yybm[0+yych] & 128) { - goto yy69; + goto yy72; + } + if (yych <= '\f') { + if (yych != '\n') goto yy3; + } else { + if (yych <= '\r') goto yy75; + if (yych == '#') goto yy67; + goto yy3; } - if (yych == '\n') goto yy71; - if (yych == '#') goto yy64; - goto yy3; -yy71: + yych = *++p; + goto yy8; +yy75: ++p; - yych = *p; - goto yy7; + if ((yych = *p) == '\n') goto yy65; + goto yy69; } } @@ -427,6 +446,7 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; @@ -468,39 +488,48 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yych <= ' ') { - if (yych <= 0x00) goto yy78; - if (yych <= 0x1F) goto yy80; + if (yych <= 0x00) goto yy82; + if (yych <= 0x1F) goto yy84; } else { - if (yych == '$') goto yy76; - goto yy80; + if (yych == '$') goto yy80; + goto yy84; } ++p; yych = *p; - goto yy84; -yy75: + goto yy92; +yy79: { continue; } -yy76: - ++p; - if ((yych = *p) == '\n') goto yy81; -yy77: +yy80: + yych = *(q = ++p); + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; +yy81: { break; } -yy78: +yy82: ++p; { break; } -yy80: +yy84: yych = *++p; - goto yy77; -yy81: + goto yy81; +yy85: ++p; { continue; } -yy83: +yy87: + yych = *++p; + if (yych == '\n') goto yy89; + p = q; + goto yy81; +yy89: + ++p; + { continue; } +yy91: ++p; yych = *p; -yy84: +yy92: if (yybm[0+yych] & 128) { - goto yy83; + goto yy91; } - goto yy75; + goto yy79; } } @@ -550,40 +579,40 @@ bool Lexer::ReadIdent(string* out) { yych = *p; if (yych <= '@') { if (yych <= '.') { - if (yych <= ',') goto yy89; + if (yych <= ',') goto yy97; } else { - if (yych <= '/') goto yy89; - if (yych >= ':') goto yy89; + if (yych <= '/') goto yy97; + if (yych >= ':') goto yy97; } } else { if (yych <= '_') { - if (yych <= 'Z') goto yy87; - if (yych <= '^') goto yy89; + if (yych <= 'Z') goto yy95; + if (yych <= '^') goto yy97; } else { - if (yych <= '`') goto yy89; - if (yych >= '{') goto yy89; + if (yych <= '`') goto yy97; + if (yych >= '{') goto yy97; } } -yy87: +yy95: ++p; yych = *p; - goto yy92; -yy88: + goto yy100; +yy96: { out->assign(start, p - start); break; } -yy89: +yy97: ++p; { return false; } -yy91: +yy99: ++p; yych = *p; -yy92: +yy100: if (yybm[0+yych] & 128) { - goto yy91; + goto yy99; } - goto yy88; + goto yy96; } } @@ -638,29 +667,36 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { yych = *p; if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy101; - if (yych >= '\n') goto yy97; + if (yych <= 0x00) goto yy110; + if (yych >= '\n') goto yy107; } else { - if (yych == '\r') goto yy103; - if (yych >= ' ') goto yy97; + if (yych == '\r') goto yy105; + if (yych >= ' ') goto yy107; } } else { if (yych <= '9') { - if (yych == '$') goto yy99; + if (yych == '$') goto yy109; } else { - if (yych <= ':') goto yy97; - if (yych == '|') goto yy97; + if (yych <= ':') goto yy107; + if (yych == '|') goto yy107; } } ++p; yych = *p; - goto yy126; -yy96: + goto yy140; +yy104: { eval->AddText(StringPiece(start, p - start)); continue; } -yy97: +yy105: + ++p; + if ((yych = *p) == '\n') goto yy137; + { + last_token_ = start; + return Error(DescribeLastError(), err); + } +yy107: ++p; { if (path) { @@ -673,137 +709,152 @@ yy97: continue; } } -yy99: - ++p; - if ((yych = *p) <= '/') { - if (yych <= ' ') { - if (yych == '\n') goto yy115; - if (yych <= 0x1F) goto yy104; - goto yy106; +yy109: + yych = *++p; + if (yych <= '-') { + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= '\t') goto yy112; + goto yy124; + } else { + if (yych == '\r') goto yy114; + goto yy112; + } } else { - if (yych <= '$') { - if (yych <= '#') goto yy104; - goto yy108; + if (yych <= '#') { + if (yych <= ' ') goto yy115; + goto yy112; } else { - if (yych == '-') goto yy110; - goto yy104; + if (yych <= '$') goto yy117; + if (yych <= ',') goto yy112; + goto yy119; } } } else { - if (yych <= '^') { - if (yych <= ':') { - if (yych <= '9') goto yy110; - goto yy112; + if (yych <= 'Z') { + if (yych <= '9') { + if (yych <= '/') goto yy112; + goto yy119; } else { - if (yych <= '@') goto yy104; - if (yych <= 'Z') goto yy110; - goto yy104; + if (yych <= ':') goto yy121; + if (yych <= '@') goto yy112; + goto yy119; } } else { if (yych <= '`') { - if (yych <= '_') goto yy110; - goto yy104; + if (yych == '_') goto yy119; + goto yy112; } else { - if (yych <= 'z') goto yy110; - if (yych <= '{') goto yy114; - goto yy104; + if (yych <= 'z') goto yy119; + if (yych <= '{') goto yy123; + goto yy112; } } } -yy100: - { - last_token_ = start; - return Error(DescribeLastError(), err); - } -yy101: +yy110: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy103: - yych = *++p; - goto yy100; -yy104: +yy112: ++p; -yy105: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy106: +yy114: + yych = *++p; + if (yych == '\n') goto yy134; + goto yy113; +yy115: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy108: +yy117: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy110: +yy119: ++p; yych = *p; - goto yy124; -yy111: + goto yy133; +yy120: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy112: +yy121: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy114: +yy123: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - goto yy105; -yy115: + goto yy113; +yy124: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy115; + goto yy124; } { continue; } -yy118: +yy127: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - if (yych == '}') goto yy121; + if (yych == '}') goto yy130; p = q; - goto yy105; -yy121: + goto yy113; +yy130: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy123: +yy132: ++p; yych = *p; -yy124: +yy133: if (yybm[0+yych] & 64) { - goto yy123; + goto yy132; } - goto yy111; -yy125: + goto yy120; +yy134: ++p; yych = *p; -yy126: + if (yych == ' ') goto yy134; + { + continue; + } +yy137: + ++p; + { + if (path) + p = start; + break; + } +yy139: + ++p; + yych = *p; +yy140: if (yybm[0+yych] & 128) { - goto yy125; + goto yy139; } - goto yy96; + goto yy104; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 93d5540..f861239 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -102,8 +102,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -132,8 +130,9 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\r\n]*"\n" { continue; } - [ ]*[\n] { token = NEWLINE; break; } + [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"\r\n" { token = NEWLINE; break; } + [ ]*"\n" { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } "pool" { token = POOL; break; } @@ -168,13 +167,15 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; /*!re2c - [ ]+ { continue; } - "$\n" { continue; } - nul { break; } - [^] { break; } + [ ]+ { continue; } + "$\r\n" { continue; } + "$\n" { continue; } + nul { break; } + [^] { break; } */ } } @@ -207,6 +208,11 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(start, p - start)); continue; } + "\r\n" { + if (path) + p = start; + break; + } [ :|\n] { if (path) { p = start; @@ -226,6 +232,9 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(" ", 1)); continue; } + "$\r\n"[ ]* { + continue; + } "$\n"[ ]* { continue; } diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 20168df..aa3e9e4 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -870,22 +870,18 @@ TEST_F(ParserTest, UTF8) { " description = compilaci\xC3\xB3\n")); } -// We might want to eventually allow CRLF to be nice to Windows developers, -// but for now just verify we error out with a nice message. TEST_F(ParserTest, CRLF) { State state; ManifestParser parser(&state, NULL); string err; - EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", - &err)); - EXPECT_EQ("input:1: lexing error\n", - err); - - EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", - &err)); - EXPECT_EQ("input:2: carriage returns are not allowed, use newlines\n" - "bar = bar\r\n" - " ^ near here", - err); + EXPECT_TRUE(parser.ParseTest("# comment with crlf\r\n", &err)); + EXPECT_TRUE(parser.ParseTest("foo = foo\nbar = bar\r\n", &err)); + EXPECT_TRUE(parser.ParseTest( + "pool link_pool\r\n" + " depth = 15\r\n\r\n" + "rule xyz\r\n" + " command = something$expand \r\n" + " description = YAY!\r\n", + &err)); } diff --git a/src/util.cc b/src/util.cc index 8b69b71..66ef113 100644 --- a/src/util.cc +++ b/src/util.cc @@ -106,6 +106,12 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { return false; } +#ifdef _WIN32 + for (char* c = path; *c; ++c) + if (*c == '\\') + *c = '/'; +#endif + const int kMaxPathComponents = 30; char* components[kMaxPathComponents]; int component_count = 0; @@ -280,7 +286,38 @@ void GetWin32EscapedString(const string& input, string* result) { } int ReadFile(const string& path, string* contents, string* err) { - FILE* f = fopen(path.c_str(), "r"); +#ifdef _WIN32 + // This makes a ninja run on a set of 1500 manifest files about 4% faster + // than using the generic fopen code below. + err->clear(); + HANDLE f = ::CreateFile(path.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_FLAG_SEQUENTIAL_SCAN, + NULL); + if (f == INVALID_HANDLE_VALUE) { + err->assign(GetLastErrorString()); + return -ENOENT; + } + + for (;;) { + DWORD len; + char buf[64 << 10]; + if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { + err->assign(GetLastErrorString()); + contents->clear(); + return -1; + } + if (len == 0) + break; + contents->append(buf, len); + } + ::CloseHandle(f); + return 0; +#else + FILE* f = fopen(path.c_str(), "rb"); if (!f) { err->assign(strerror(errno)); return -errno; @@ -299,6 +336,7 @@ int ReadFile(const string& path, string* contents, string* err) { } fclose(f); return 0; +#endif } void SetCloseOnExec(int fd) { diff --git a/src/util_test.cc b/src/util_test.cc index b58d15e..e82f227 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -84,6 +84,72 @@ TEST(CanonicalizePath, PathSamples) { EXPECT_EQ("", path); } +#ifdef _WIN32 +TEST(CanonicalizePath, PathSamplesWindows) { + string path; + string err; + + EXPECT_FALSE(CanonicalizePath(&path, &err)); + EXPECT_EQ("empty path", err); + + path = "foo.h"; err = ""; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo\\.\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar.h", path); + + path = ".\\x\\foo\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("x/bar.h", path); + + path = ".\\x\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar.h", path); + + path = "foo\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar", path); + + path = "foo\\\\.\\\\..\\\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar", path); + + path = ".\\x\\..\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("../bar.h", path); + + path = "foo\\.\\."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\bar\\.."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\.hidden_bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/.hidden_bar", path); + + path = "\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("/foo", path); + + path = "\\\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("//foo", path); + + path = "\\"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("", path); +} +#endif + TEST(CanonicalizePath, EmptyResult) { string path; string err; -- cgit v0.12