From 3a889126247d31d637c7470554be12f7c1df2ffa Mon Sep 17 00:00:00 2001 From: Nico Weber Date: Thu, 11 Jun 2015 23:53:32 -0700 Subject: Make deps=msvc experimentally available on non-Windows. This makes it possible to run most of the clparser tests on non-Windows, and is potentially useful for cross-compiling on non-Windows hosts. Also, the manual didn't document this as Windows-only previously. If you use this on non-Windows, please let me know, else I might undo this change again in the future. --- configure.py | 2 + src/build.cc | 4 +- src/clparser.cc | 116 +++++++++++++++++++++++++++++++++++++++++ src/clparser.h | 52 +++++++++++++++++++ src/clparser_test.cc | 117 ++++++++++++++++++++++++++++++++++++++++++ src/msvc_helper-win32.cc | 83 ------------------------------ src/msvc_helper.h | 32 ------------ src/msvc_helper_main-win32.cc | 1 + src/msvc_helper_test.cc | 94 +-------------------------------- 9 files changed, 290 insertions(+), 211 deletions(-) create mode 100644 src/clparser.cc create mode 100644 src/clparser.h create mode 100644 src/clparser_test.cc diff --git a/configure.py b/configure.py index f0e452f..84218b9 100755 --- a/configure.py +++ b/configure.py @@ -475,6 +475,7 @@ n.comment('Core source files all build into ninja library.') for name in ['build', 'build_log', 'clean', + 'clparser', 'debug_flags', 'depfile_parser', 'deps_log', @@ -540,6 +541,7 @@ objs = [] for name in ['build_log_test', 'build_test', 'clean_test', + 'clparser_test', 'depfile_parser_test', 'deps_log_test', 'disk_interface_test', diff --git a/src/build.cc b/src/build.cc index e33a007..2df5c1a 100644 --- a/src/build.cc +++ b/src/build.cc @@ -25,12 +25,12 @@ #endif #include "build_log.h" +#include "clparser.h" #include "debug_flags.h" #include "depfile_parser.h" #include "deps_log.h" #include "disk_interface.h" #include "graph.h" -#include "msvc_helper.h" #include "state.h" #include "subprocess.h" #include "util.h" @@ -854,7 +854,6 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, const string& deps_prefix, vector* deps_nodes, string* err) { -#ifdef _WIN32 if (deps_type == "msvc") { CLParser parser; string output; @@ -870,7 +869,6 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, deps_nodes->push_back(state_->GetNode(*i, ~0u)); } } else -#endif if (deps_type == "gcc") { string depfile = result->edge->GetUnescapedDepfile(); if (depfile.empty()) { diff --git a/src/clparser.cc b/src/clparser.cc new file mode 100644 index 0000000..f73a8c1 --- /dev/null +++ b/src/clparser.cc @@ -0,0 +1,116 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "clparser.h" + +#include +#include +#include + +#ifdef _WIN32 +#include "includes_normalize.h" +#else +#include "util.h" +#endif + +namespace { + +/// Return true if \a input ends with \a needle. +bool EndsWith(const string& input, const string& needle) { + return (input.size() >= needle.size() && + input.substr(input.size() - needle.size()) == needle); +} + +} // anonymous namespace + +// static +string CLParser::FilterShowIncludes(const string& line, + const string& deps_prefix) { + const string kDepsPrefixEnglish = "Note: including file: "; + const char* in = line.c_str(); + const char* end = in + line.size(); + const string& prefix = deps_prefix.empty() ? kDepsPrefixEnglish : deps_prefix; + if (end - in > (int)prefix.size() && + memcmp(in, prefix.c_str(), (int)prefix.size()) == 0) { + in += prefix.size(); + while (*in == ' ') + ++in; + return line.substr(in - line.c_str()); + } + return ""; +} + +// static +bool CLParser::IsSystemInclude(string path) { + transform(path.begin(), path.end(), path.begin(), ::tolower); + // TODO: this is a heuristic, perhaps there's a better way? + return (path.find("program files") != string::npos || + path.find("microsoft visual studio") != string::npos); +} + +// static +bool CLParser::FilterInputFilename(string line) { + transform(line.begin(), line.end(), line.begin(), ::tolower); + // TODO: other extensions, like .asm? + return EndsWith(line, ".c") || + EndsWith(line, ".cc") || + EndsWith(line, ".cxx") || + EndsWith(line, ".cpp"); +} + +// static +bool CLParser::Parse(const string& output, const string& deps_prefix, + string* filtered_output, string* err) { + // Loop over all lines in the output to process them. + assert(&output != filtered_output); + size_t start = 0; + while (start < output.size()) { + size_t end = output.find_first_of("\r\n", start); + if (end == string::npos) + end = output.size(); + string line = output.substr(start, end - start); + + string include = FilterShowIncludes(line, deps_prefix); + if (!include.empty()) { + string normalized; +#ifdef _WIN32 + if (!IncludesNormalize::Normalize(include, NULL, &normalized, err)) + return false; +#else + // TODO: should this make the path relative to cwd? + normalized = include; + unsigned int slash_bits; + if (!CanonicalizePath(&normalized, &slash_bits, err)) + return false; +#endif + if (!IsSystemInclude(normalized)) + includes_.insert(normalized); + } else if (FilterInputFilename(line)) { + // Drop it. + // TODO: if we support compiling multiple output files in a single + // cl.exe invocation, we should stash the filename. + } else { + filtered_output->append(line); + filtered_output->append("\n"); + } + + if (end < output.size() && output[end] == '\r') + ++end; + if (end < output.size() && output[end] == '\n') + ++end; + start = end; + } + + return true; +} diff --git a/src/clparser.h b/src/clparser.h new file mode 100644 index 0000000..e597e7e --- /dev/null +++ b/src/clparser.h @@ -0,0 +1,52 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_CLPARSER_H_ +#define NINJA_CLPARSER_H_ + +#include +#include +using namespace std; + +/// Visual Studio's cl.exe requires some massaging to work with Ninja; +/// for example, it emits include information on stderr in a funny +/// format when building with /showIncludes. This class parses this +/// output. +struct CLParser { + /// Parse a line of cl.exe output and extract /showIncludes info. + /// If a dependency is extracted, returns a nonempty string. + /// Exposed for testing. + static string FilterShowIncludes(const string& line, + const string& deps_prefix); + + /// Return true if a mentioned include file is a system path. + /// Filtering these out reduces dependency information considerably. + static bool IsSystemInclude(string path); + + /// Parse a line of cl.exe output and return true if it looks like + /// it's printing an input filename. This is a heuristic but it appears + /// to be the best we can do. + /// Exposed for testing. + static bool FilterInputFilename(string line); + + /// Parse the full output of cl, filling filtered_output with the text that + /// should be printed (if any). Returns true on success, or false with err + /// filled. output must not be the same object as filtered_object. + bool Parse(const string& output, const string& deps_prefix, + string* filtered_output, string* err); + + set includes_; +}; + +#endif // NINJA_CLPARSER_H_ diff --git a/src/clparser_test.cc b/src/clparser_test.cc new file mode 100644 index 0000000..1549ab1 --- /dev/null +++ b/src/clparser_test.cc @@ -0,0 +1,117 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "clparser.h" + +#include "test.h" +#include "util.h" + +TEST(CLParserTest, ShowIncludes) { + ASSERT_EQ("", CLParser::FilterShowIncludes("", "")); + + ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output", "")); + ASSERT_EQ("c:\\Some Files\\foobar.h", + CLParser::FilterShowIncludes("Note: including file: " + "c:\\Some Files\\foobar.h", "")); + ASSERT_EQ("c:\\initspaces.h", + CLParser::FilterShowIncludes("Note: including file: " + "c:\\initspaces.h", "")); + ASSERT_EQ("c:\\initspaces.h", + CLParser::FilterShowIncludes("Non-default prefix: inc file: " + "c:\\initspaces.h", + "Non-default prefix: inc file:")); +} + +TEST(CLParserTest, FilterInputFilename) { + ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc")); + ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc")); + ASSERT_TRUE(CLParser::FilterInputFilename("baz.c")); + ASSERT_TRUE(CLParser::FilterInputFilename("FOOBAR.CC")); + + ASSERT_FALSE(CLParser::FilterInputFilename( + "src\\cl_helper.cc(166) : fatal error C1075: end " + "of file found ...")); +} + +TEST(CLParserTest, ParseSimple) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "foo\r\n" + "Note: inc file prefix: foo.h\r\n" + "bar\r\n", + "Note: inc file prefix:", &output, &err)); + + ASSERT_EQ("foo\nbar\n", output); + ASSERT_EQ(1u, parser.includes_.size()); + ASSERT_EQ("foo.h", *parser.includes_.begin()); +} + +TEST(CLParserTest, ParseFilenameFilter) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "foo.cc\r\n" + "cl: warning\r\n", + "", &output, &err)); + ASSERT_EQ("cl: warning\n", output); +} + +TEST(CLParserTest, ParseSystemInclude) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "Note: including file: c:\\Program Files\\foo.h\r\n" + "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n" + "Note: including file: path.h\r\n", + "", &output, &err)); + // We should have dropped the first two includes because they look like + // system headers. + ASSERT_EQ("", output); + ASSERT_EQ(1u, parser.includes_.size()); + ASSERT_EQ("path.h", *parser.includes_.begin()); +} + +TEST(CLParserTest, DuplicatedHeader) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "Note: including file: foo.h\r\n" + "Note: including file: bar.h\r\n" + "Note: including file: foo.h\r\n", + "", &output, &err)); + // We should have dropped one copy of foo.h. + ASSERT_EQ("", output); + ASSERT_EQ(2u, parser.includes_.size()); +} + +TEST(CLParserTest, DuplicatedHeaderPathConverted) { + CLParser parser; + string output, err; + + // This isn't inline in the Parse() call below because the #ifdef in + // a macro expansion would confuse MSVC2013's preprocessor. + const char kInput[] = + "Note: including file: sub/./foo.h\r\n" + "Note: including file: bar.h\r\n" +#ifdef _WIN32 + "Note: including file: sub\\foo.h\r\n"; +#else + "Note: including file: sub/foo.h\r\n"; +#endif + ASSERT_TRUE(parser.Parse(kInput, "", &output, &err)); + // We should have dropped one copy of foo.h. + ASSERT_EQ("", output); + ASSERT_EQ(2u, parser.includes_.size()); +} diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index d516240..e37a26e 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -14,23 +14,12 @@ #include "msvc_helper.h" -#include -#include -#include -#include #include -#include "includes_normalize.h" #include "util.h" namespace { -/// Return true if \a input ends with \a needle. -bool EndsWith(const string& input, const string& needle) { - return (input.size() >= needle.size() && - input.substr(input.size() - needle.size()) == needle); -} - string Replace(const string& input, const string& find, const string& replace) { string result = input; size_t start_pos = 0; @@ -48,78 +37,6 @@ string EscapeForDepfile(const string& path) { return Replace(path, " ", "\\ "); } -// static -string CLParser::FilterShowIncludes(const string& line, - const string& deps_prefix) { - const string kDepsPrefixEnglish = "Note: including file: "; - const char* in = line.c_str(); - const char* end = in + line.size(); - const string& prefix = deps_prefix.empty() ? kDepsPrefixEnglish : deps_prefix; - if (end - in > (int)prefix.size() && - memcmp(in, prefix.c_str(), (int)prefix.size()) == 0) { - in += prefix.size(); - while (*in == ' ') - ++in; - return line.substr(in - line.c_str()); - } - return ""; -} - -// static -bool CLParser::IsSystemInclude(string path) { - transform(path.begin(), path.end(), path.begin(), ::tolower); - // TODO: this is a heuristic, perhaps there's a better way? - return (path.find("program files") != string::npos || - path.find("microsoft visual studio") != string::npos); -} - -// static -bool CLParser::FilterInputFilename(string line) { - transform(line.begin(), line.end(), line.begin(), ::tolower); - // TODO: other extensions, like .asm? - return EndsWith(line, ".c") || - EndsWith(line, ".cc") || - EndsWith(line, ".cxx") || - EndsWith(line, ".cpp"); -} - -bool CLParser::Parse(const string& output, const string& deps_prefix, - string* filtered_output, string* err) { - // Loop over all lines in the output to process them. - assert(&output != filtered_output); - size_t start = 0; - while (start < output.size()) { - size_t end = output.find_first_of("\r\n", start); - if (end == string::npos) - end = output.size(); - string line = output.substr(start, end - start); - - string include = FilterShowIncludes(line, deps_prefix); - if (!include.empty()) { - string normalized; - if (!IncludesNormalize::Normalize(include, NULL, &normalized, err)) - return false; - if (!IsSystemInclude(normalized)) - includes_.insert(normalized); - } else if (FilterInputFilename(line)) { - // Drop it. - // TODO: if we support compiling multiple output files in a single - // cl.exe invocation, we should stash the filename. - } else { - filtered_output->append(line); - filtered_output->append("\n"); - } - - if (end < output.size() && output[end] == '\r') - ++end; - if (end < output.size() && output[end] == '\n') - ++end; - start = end; - } - - return true; -} - int CLWrapper::Run(const string& command, string* output) { SECURITY_ATTRIBUTES security_attributes = {}; security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 30f87f3..70d1fff 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -13,42 +13,10 @@ // limitations under the License. #include -#include -#include using namespace std; string EscapeForDepfile(const string& path); -/// Visual Studio's cl.exe requires some massaging to work with Ninja; -/// for example, it emits include information on stderr in a funny -/// format when building with /showIncludes. This class parses this -/// output. -struct CLParser { - /// Parse a line of cl.exe output and extract /showIncludes info. - /// If a dependency is extracted, returns a nonempty string. - /// Exposed for testing. - static string FilterShowIncludes(const string& line, - const string& deps_prefix); - - /// Return true if a mentioned include file is a system path. - /// Filtering these out reduces dependency information considerably. - static bool IsSystemInclude(string path); - - /// Parse a line of cl.exe output and return true if it looks like - /// it's printing an input filename. This is a heuristic but it appears - /// to be the best we can do. - /// Exposed for testing. - static bool FilterInputFilename(string line); - - /// Parse the full output of cl, filling filtered_output with the text that - /// should be printed (if any). Returns true on success, or false with err - /// filled. output must not be the same object as filtered_object. - bool Parse(const string& output, const string& deps_prefix, - string* filtered_output, string* err); - - set includes_; -}; - /// Wraps a synchronous execution of a CL subprocess. struct CLWrapper { CLWrapper() : env_block_(NULL) {} diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 680aaad..e419cd7 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -19,6 +19,7 @@ #include #include +#include "clparser.h" #include "util.h" #include "getopt.h" diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 49d27c1..eaae51f 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -17,99 +17,7 @@ #include "test.h" #include "util.h" -TEST(CLParserTest, ShowIncludes) { - ASSERT_EQ("", CLParser::FilterShowIncludes("", "")); - - ASSERT_EQ("", CLParser::FilterShowIncludes("Sample compiler output", "")); - ASSERT_EQ("c:\\Some Files\\foobar.h", - CLParser::FilterShowIncludes("Note: including file: " - "c:\\Some Files\\foobar.h", "")); - ASSERT_EQ("c:\\initspaces.h", - CLParser::FilterShowIncludes("Note: including file: " - "c:\\initspaces.h", "")); - ASSERT_EQ("c:\\initspaces.h", - CLParser::FilterShowIncludes("Non-default prefix: inc file: " - "c:\\initspaces.h", - "Non-default prefix: inc file:")); -} - -TEST(CLParserTest, FilterInputFilename) { - ASSERT_TRUE(CLParser::FilterInputFilename("foobar.cc")); - ASSERT_TRUE(CLParser::FilterInputFilename("foo bar.cc")); - ASSERT_TRUE(CLParser::FilterInputFilename("baz.c")); - ASSERT_TRUE(CLParser::FilterInputFilename("FOOBAR.CC")); - - ASSERT_FALSE(CLParser::FilterInputFilename( - "src\\cl_helper.cc(166) : fatal error C1075: end " - "of file found ...")); -} - -TEST(CLParserTest, ParseSimple) { - CLParser parser; - string output, err; - ASSERT_TRUE(parser.Parse( - "foo\r\n" - "Note: inc file prefix: foo.h\r\n" - "bar\r\n", - "Note: inc file prefix:", &output, &err)); - - ASSERT_EQ("foo\nbar\n", output); - ASSERT_EQ(1u, parser.includes_.size()); - ASSERT_EQ("foo.h", *parser.includes_.begin()); -} - -TEST(CLParserTest, ParseFilenameFilter) { - CLParser parser; - string output, err; - ASSERT_TRUE(parser.Parse( - "foo.cc\r\n" - "cl: warning\r\n", - "", &output, &err)); - ASSERT_EQ("cl: warning\n", output); -} - -TEST(CLParserTest, ParseSystemInclude) { - CLParser parser; - string output, err; - ASSERT_TRUE(parser.Parse( - "Note: including file: c:\\Program Files\\foo.h\r\n" - "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n" - "Note: including file: path.h\r\n", - "", &output, &err)); - // We should have dropped the first two includes because they look like - // system headers. - ASSERT_EQ("", output); - ASSERT_EQ(1u, parser.includes_.size()); - ASSERT_EQ("path.h", *parser.includes_.begin()); -} - -TEST(CLParserTest, DuplicatedHeader) { - CLParser parser; - string output, err; - ASSERT_TRUE(parser.Parse( - "Note: including file: foo.h\r\n" - "Note: including file: bar.h\r\n" - "Note: including file: foo.h\r\n", - "", &output, &err)); - // We should have dropped one copy of foo.h. - ASSERT_EQ("", output); - ASSERT_EQ(2u, parser.includes_.size()); -} - -TEST(CLParserTest, DuplicatedHeaderPathConverted) { - CLParser parser; - string output, err; - ASSERT_TRUE(parser.Parse( - "Note: including file: sub/foo.h\r\n" - "Note: including file: bar.h\r\n" - "Note: including file: sub\\foo.h\r\n", - "", &output, &err)); - // We should have dropped one copy of foo.h. - ASSERT_EQ("", output); - ASSERT_EQ(2u, parser.includes_.size()); -} - -TEST(CLParserTest, SpacesInFilename) { +TEST(EscapeForDepfileTest, SpacesInFilename) { ASSERT_EQ("sub\\some\\ sdk\\foo.h", EscapeForDepfile("sub\\some sdk\\foo.h")); } -- cgit v0.12