diff options
-rw-r--r-- | .gitignore | 32 | ||||
-rw-r--r-- | HACKING | 76 | ||||
-rw-r--r-- | HACKING.md | 165 | ||||
-rwxr-xr-x | bootstrap.py | 57 | ||||
-rwxr-xr-x | configure.py | 72 | ||||
-rw-r--r-- | doc/manual.asciidoc | 4 | ||||
-rw-r--r-- | src/build_log.cc | 12 | ||||
-rw-r--r-- | src/build_test.cc | 56 | ||||
-rw-r--r-- | src/canon_perftest.cc | 2 | ||||
-rw-r--r-- | src/depfile_parser.cc | 4 | ||||
-rw-r--r-- | src/depfile_parser.in.cc | 4 | ||||
-rw-r--r-- | src/graph.cc | 13 | ||||
-rw-r--r-- | src/graph.h | 8 | ||||
-rw-r--r-- | src/hash_collision_bench.cc | 9 | ||||
-rw-r--r-- | src/hash_map.h | 2 | ||||
-rw-r--r-- | src/includes_normalize-win32.cc | 115 | ||||
-rw-r--r-- | src/includes_normalize.h | 35 | ||||
-rw-r--r-- | src/includes_normalize_test.cc | 98 | ||||
-rw-r--r-- | src/lexer.cc | 2 | ||||
-rw-r--r-- | src/lexer.in.cc | 2 | ||||
-rw-r--r-- | src/msvc_helper-win32.cc | 164 | ||||
-rw-r--r-- | src/msvc_helper.h | 54 | ||||
-rw-r--r-- | src/msvc_helper_main-win32.cc | 115 | ||||
-rw-r--r-- | src/msvc_helper_test.cc | 83 | ||||
-rw-r--r-- | src/ninja.cc | 17 | ||||
-rw-r--r-- | src/string_piece.h | 4 | ||||
-rw-r--r-- | src/subprocess-win32.cc | 8 | ||||
-rw-r--r-- | src/util.cc | 10 | ||||
-rw-r--r-- | src/util.h | 8 | ||||
-rw-r--r-- | src/util_test.cc | 6 |
30 files changed, 1066 insertions, 171 deletions
@@ -1,15 +1,17 @@ -*.pyc -*.exe -TAGS -/build -/build.ninja -/ninja -/build_log_perftest -/canon_perftest -/hash_collision_bench -/ninja_test -/parser_perftest -/graph.png -/doc/manual.html -/doc/doxygen -/gtest-1.6.0 +*.pyc
+*.exe
+*.pdb
+*.ilk
+TAGS
+/build
+/build.ninja
+/ninja
+/build_log_perftest
+/canon_perftest
+/hash_collision_bench
+/ninja_test
+/parser_perftest
+/graph.png
+/doc/manual.html
+/doc/doxygen
+/gtest-1.6.0
diff --git a/HACKING b/HACKING deleted file mode 100644 index 4e01d49..0000000 --- a/HACKING +++ /dev/null @@ -1,76 +0,0 @@ -Adjusting build flags: - CFLAGS=-O3 ./configure.py - and rebuild. - -Building tests requires gtest, to get it: - - On older Ubuntus you can apt-get install libgtest. - - On newer Ubuntus it's only distributed as source; - 1) apt-get install libgtest-dev - 2) ./configure --with-gtest=/usr/src/gtest - - Otherwise you need to download it, unpack it, and pass --with-gtest - as appropriate. - -Test-driven development: - Set your build command to - ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name - now you can repeatedly run that while developing until the tests pass. - Remember to build "all" before committing to verify the other source - still works! - -Testing performance impact of changes: - If you have a Chrome build handy, it's a good test case. - Otherwise, https://github.com/martine/ninja/downloads has a copy of - the Chrome build files (and depfiles). You can untar that, then run - path/to/my/ninja chrome - and compare that against a baseline Ninja. - - There's a script at misc/measure.py that repeatedly runs a command like - the above (to address variance) and summarizes its runtime. E.g. - path/to/misc/measure.py path/to/my/ninja chrome - - For changing the depfile parser, you can also build 'parser_perftest' - and run that directly on some representative input files. - -Coding guidelines: -- Function name are camelcase. -- Member methods are camelcase, expect for trivial getters which are - underscore separated. -- Local variables are underscore separated. -- Member variables are underscore separated and suffixed by an extra underscore. -- Two spaces indentation. -- Opening braces is at the end of line. -- Lines are 80 columns maximum. -- All source files should have the Google Inc. license header. -- Also follow this style: - http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml - -Documentation guidelines: -- Use /// for doxygen. -- Use \a to refer to arguments. -- It's not necessary to document each argument, especially when they're - relatively self-evident (e.g. in CanonicalizePath(string* path, string* err), - the arguments are hopefully obvious) - -Generating the manual: - sudo apt-get install asciidoc --no-install-recommends - ./ninja manual - -Windows development on Linux (this is kind of hacky right now): -- Get the gtest source, unpack it into your source dir -- sudo apt-get install gcc-mingw32 wine -- export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar -- ./configure.py --platform=mingw --host=linux --with-gtest=gtest-1.6.0 -- Build ninja: /path/to/linux/ninja -- Run: ./ninja.exe (implicitly runs through wine(!)) - -Windows development on Windows: -- install mingw, msys, and python -- in the mingw shell, put Python in your path, and: python bootstrap.py -- to reconfigure, run 'python configure.py' -- remember to strip the resulting executable if size matters to you -- you'll need to rename ninja.exe into my-ninja.exe during development, - otherwise ninja won't be able to overwrite itself when building - -Using clang: -- Enable colors manually: - CXX='/path/to/llvm/Release+Asserts/bin/clang++ -fcolor-diagnostics' ./configure.py diff --git a/HACKING.md b/HACKING.md new file mode 100644 index 0000000..bc23773 --- /dev/null +++ b/HACKING.md @@ -0,0 +1,165 @@ +## Basic overview + +`./configure.py` generates the `build.ninja` files used to build +ninja. It accepts various flags to adjust build parameters. + +The primary build target of interest is `ninja`, but when hacking on +Ninja your changes should be testable so it's more useful to build +and run `ninja_test` when developing. + +(`./bootstrap.py` creates a bootstrap `ninja` and runs the above +process; it's only necessary to run if you don't have a copy of +`ninja` to build with.) + +### Adjusting build flags + +Build in "debug" mode while developing (disables optimizations and builds +way faster on Windows): + + ./configure.py --debug + +To use clang, set `CXX`: + + CXX=clang++ ./configure.py + +## How to successfully make changes to Ninja + +Github pull requests are convenient for me to merge (I can just click +a button and it's all handled server-side), but I'm also comfortable +accepting pre-github git patches (via `send-email` etc.). + +Good pull requests have all of these attributes: + +* Are scoped to one specific issue +* Include a test to demonstrate their correctness +* Update the docs where relevant +* Match the Ninja coding style (see below) +* Don't include a mess of "oops, fix typo" commits + +These are typically merged without hesitation. If a change is lacking +any of the above I usually will ask you to fix it, though there are +obvious exceptions (fixing typos in comments don't need tests). + +I am very wary of changes that increase the complexity of Ninja (in +particular, new build file syntax or command-line flags) or increase +the maintenance burden of Ninja. Ninja is already successfully in use +by hundreds of developers for large projects and it already achieves +(most of) the goals I set out for it to do. It's probably best to +discuss new feature ideas on the mailing list before I shoot down your +patch. + +## Testing + +### Installing gtest + +The `ninja_test` binary, containing all the tests, depends on the +googletest (gtest) library. + +* On older Ubuntus it'll install as libraries into `/usr/lib`: + + apt-get install libgtest + +* On newer Ubuntus it's only distributed as source + + apt-get install libgtest-dev + ./configure --with-gtest=/usr/src/gtest + +* Otherwise you need to download it, unpack it, and pass + `--with-gtest` to `configure.py`. Get it from [its downloads + page](http://code.google.com/p/googletest/downloads/list). + +### Test-driven development + +Set your build command to + + ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name + +now you can repeatedly run that while developing until the tests pass +(I frequently set it as my compilation command in Emacs). Remember to +build "all" before committing to verify the other source still works! + +## Testing performance impact of changes + +If you have a Chrome build handy, it's a good test case. Otherwise, +[the github downoads page](https://github.com/martine/ninja/downloads) +has a copy of the Chrome build files (and depfiles). You can untar +that, then run + + path/to/my/ninja chrome + +and compare that against a baseline Ninja. + +There's a script at `misc/measure.py` that repeatedly runs a command like +the above (to address variance) and summarizes its runtime. E.g. + + path/to/misc/measure.py path/to/my/ninja chrome + +For changing the depfile parser, you can also build `parser_perftest` +and run that directly on some representative input files. + +## Coding guidelines + +Generally it's the [Google C++ coding style][], but in brief: + +* Function name are camelcase. +* Member methods are camelcase, expect for trivial getters which are + underscore separated. +* Local variables are underscore separated. +* Member variables are underscore separated and suffixed by an extra + underscore. +* Two spaces indentation. +* Opening braces is at the end of line. +* Lines are 80 columns maximum. +* All source files should have the Google Inc. license header. + +[Google C++ coding style]: http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml + +## Documentation + +### Style guidelines + +* Use `///` for doxygen. +* Use `\a` to refer to arguments. +* It's not necessary to document each argument, especially when they're + relatively self-evident (e.g. in `CanonicalizePath(string* path, string* err)`, + the arguments are hopefully obvious) + +### Building the manual + + sudo apt-get install asciidoc --no-install-recommends + ./ninja manual + +### Building the code documentation + + sudo apt-get install doxygen + ./ninja doxygen + +## Building for Windows + +While developing, it's helpful to copy `ninja.exe` to another name like +`n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because +it's locked while in use. + +### Via Visual Studio + +* Install Visual Studio (Express is fine), [Python for Windows][], + and (if making changes) googletest (see above instructions) +* In a Visual Studio command prompt: `python bootstrap.py` + +[Python for Windows]: http://www.python.org/getit/windows/ + +### Via mingw on Windows (not well supported) + +* Install mingw, msys, and python +* In the mingw shell, put Python in your path, and `python bootstrap.py` +* To reconfigure, run `python configure.py` +* Remember to strip the resulting executable if size matters to you + +### Via mingw on Linux (not well supported) + +* `sudo apt-get install gcc-mingw32 wine` +* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar` +* `./configure.py --platform=mingw --host=linux` +* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` +* Run: `./ninja.exe` (implicitly runs through wine(!)) + diff --git a/bootstrap.py b/bootstrap.py index ab03cf8..abd2528 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -26,6 +26,8 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = OptionParser() parser.add_option('--verbose', action='store_true', help='enable verbose build',) +parser.add_option('--x64', action='store_true', + help='force 64-bit build (Windows)',) (options, conf_args) = parser.parse_args() def run(*args, **kwargs): @@ -66,6 +68,8 @@ for src in glob.glob('src/*.cc'): else: if src.endswith('-win32.cc'): continue + if '_main' in src: + continue sources.append(src) @@ -74,8 +78,11 @@ if sys.platform.startswith('win32'): vcdir = os.environ.get('VCINSTALLDIR') if vcdir: - args = [os.path.join(vcdir, 'bin', 'cl.exe'), - '/nologo', '/EHsc', '/DNOMINMAX'] + if options.x64: + cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')] + else: + cl = [os.path.join(vcdir, 'bin', 'cl.exe')] + args = cl + ['/nologo', '/EHsc', '/DNOMINMAX'] else: args = shlex.split(os.environ.get('CXX', 'g++')) cflags.extend(['-Wno-deprecated', @@ -83,6 +90,8 @@ else: '-DNINJA_BOOTSTRAP']) if sys.platform.startswith('win32'): cflags.append('-D_WIN32_WINNT=0x0501') + if options.x64: + cflags.append('-m64') args.extend(cflags) args.extend(ldflags) binary = 'ninja.bootstrap' @@ -103,9 +112,41 @@ verbose = [] if options.verbose: verbose = ['-v'] -print 'Building ninja using itself...' -run([sys.executable, 'configure.py'] + conf_args) -run(['./' + binary] + verbose) -os.unlink(binary) - -print 'Done!' +if sys.platform.startswith('win32'): + # Build ninja-msvc-helper using ninja without an msvc-helper. + print 'Building ninja-msvc-helper...' + run([sys.executable, 'configure.py', '--with-msvc-helper='] + conf_args) + run(['./' + binary] + verbose + ['ninja-msvc-helper']) + + # Rename the helper to the same name + .bootstrap. + helper_binary = 'ninja-msvc-helper.bootstrap.exe' + try: + os.unlink(helper_binary) + except: + pass + os.rename('ninja-msvc-helper.exe', helper_binary) + + # Build ninja using the newly-built msvc-helper. + print 'Building ninja using itself...' + run([sys.executable, 'configure.py', + '--with-msvc-helper=%s' % helper_binary] + conf_args) + run(['./' + binary] + verbose) + + # Clean up. + for obj in glob.glob('*.obj'): + os.unlink(obj) + + print """ +Done! + +Note: to work around Windows file locking, where you can't rebuild an +in-use binary, to run ninja after making any changes to build ninja itself +you should run ninja.bootstrap instead. Your build is also configured to +use ninja-msvc-helper.bootstrap.exe instead of the ninja-msvc-helper.exe +that it builds; see the --help output of configure.py.""" +else: + print 'Building ninja using itself...' + run([sys.executable, 'configure.py'] + conf_args) + run(['./' + binary] + verbose) + os.unlink(binary) + print 'Done!' diff --git a/configure.py b/configure.py index 95f88b1..e4b9eb4 100755 --- a/configure.py +++ b/configure.py @@ -45,6 +45,8 @@ parser.add_option('--with-gtest', metavar='PATH', parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) +parser.add_option('--with-msvc-helper', metavar='NAME', + help="name for ninja-msvc-helper binary (MSVC only)") (options, args) = parser.parse_args() if args: print 'ERROR: extra unparsed command-line arguments:', args @@ -97,7 +99,9 @@ def cxx(name, **kwargs): return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs) def binary(name): if platform in ('mingw', 'windows'): - return name + '.exe' + exe = name + '.exe' + n.build(name, 'phony', exe) + return exe return name n.variable('builddir', 'build') @@ -108,8 +112,16 @@ else: n.variable('ar', configure_env.get('AR', 'ar')) if platform == 'windows': - cflags = ['/nologo', '/Zi', '/W4', '/WX', '/wd4530', '/wd4100', '/wd4706', - '/wd4512', '/wd4800', '/wd4702', '/wd4819', '/GR-', + cflags = ['/nologo', # Don't print startup banner. + '/Zi', # Create pdb with debug info. + '/W4', # Highest warning level. + '/WX', # Warnings as errors. + '/wd4530', '/wd4100', '/wd4706', + '/wd4512', '/wd4800', '/wd4702', '/wd4819', + '/GR-', # Disable RTTI. + # Disable size_t -> int truncation warning. + # We never have strings or arrays larger than 2**31. + '/wd4267', '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', '/DNINJA_PYTHON="%s"' % options.with_python] ldflags = ['/DEBUG', '/libpath:$builddir'] @@ -167,8 +179,11 @@ n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags)) n.newline() if platform == 'windows': + compiler = '$cxx' + if options.with_msvc_helper: + compiler = '%s -o $out -- $cxx /showIncludes' % options.with_msvc_helper n.rule('cxx', - command='$cxx $cflags -c $in /Fo$out', + command='%s $cflags -c $in /Fo$out' % compiler, depfile='$out.d', description='CXX $out') else: @@ -218,12 +233,23 @@ if platform not in ('mingw', 'windows'): n.newline() n.comment('the depfile parser and ninja lexers are generated using re2c.') -n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', - description='RE2C $out') -# Generate the .cc files in the source directory so we can check them in. -n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) -n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +def has_re2c(): + import subprocess + try: + subprocess.call(['re2c', '-v'], stdout=subprocess.PIPE) + return True + except OSError: + return False +if has_re2c(): + n.rule('re2c', + command='re2c -b -i --no-generation-date -o $out $in', + description='RE2C $out') + # Generate the .cc files in the source directory so we can check them in. + n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) + n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) +else: + print ("warning: re2c not found; changes to src/*.in.cc will not affect " + "your build.") n.newline() n.comment('Core source files all build into ninja library.') @@ -246,6 +272,8 @@ for name in ['build', if platform in ('mingw', 'windows'): objs += cxx('subprocess-win32') if platform == 'windows': + objs += cxx('includes_normalize-win32') + objs += cxx('msvc_helper-win32') objs += cxx('minidump-win32') objs += cc('getopt') else: @@ -267,11 +295,19 @@ n.comment('Main executable is library plus main() function.') objs = cxx('ninja') ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) -if 'ninja' not in ninja: - n.build('ninja', 'phony', ninja) n.newline() all_targets += ninja +if platform == 'windows': + n.comment('Helper for working with MSVC.') + msvc_helper = n.build(binary('ninja-msvc-helper'), 'link', + cxx('msvc_helper_main-win32'), + implicit=ninja_lib, + variables=[('libs', libs)]) + n.default(msvc_helper) + n.newline() + all_targets += msvc_helper + n.comment('Tests all build into ninja_test executable.') variables = [] @@ -288,10 +324,10 @@ if options.with_gtest: else: gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs objs += n.build(built('gtest-all' + objext), 'cxx', - os.path.join(path, 'src/gtest-all.cc'), + os.path.join(path, 'src', 'gtest-all.cc'), variables=[('cflags', gtest_cflags)]) objs += n.build(built('gtest_main' + objext), 'cxx', - os.path.join(path, 'src/gtest_main.cc'), + os.path.join(path, 'src', 'gtest_main.cc'), variables=[('cflags', gtest_cflags)]) test_cflags = cflags + ['-DGTEST_HAS_RTTI=0', @@ -315,14 +351,15 @@ for name in ['build_log_test', 'test', 'util_test']: objs += cxx(name, variables=[('cflags', test_cflags)]) +if platform == 'windows': + for name in ['includes_normalize_test', 'msvc_helper_test']: + objs += cxx(name, variables=[('cflags', test_cflags)]) if platform != 'mingw' and platform != 'windows': test_libs.append('-lpthread') ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, variables=[('ldflags', test_ldflags), ('libs', test_libs)]) -if 'ninja_test' not in ninja_test: - n.build('ninja_test', 'phony', ninja_test) n.newline() all_targets += ninja_test @@ -370,7 +407,7 @@ n.rule('doxygen_mainpage', command='$doxygen_mainpage_generator $in > $out', description='DOXYGEN_MAINPAGE $out') mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage', - ['README', 'HACKING', 'COPYING'], + ['README', 'COPYING'], implicit=['$doxygen_mainpage_generator']) n.build('doxygen', 'doxygen', doc('doxygen.config'), implicit=mainpage) @@ -386,7 +423,6 @@ if host != 'mingw': implicit=['configure.py', os.path.normpath('misc/ninja_syntax.py')]) n.newline() -n.comment('Build only the main binary by default.') n.default(ninja) n.newline() diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 52cce3b..03d27df 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -524,7 +524,9 @@ Lexical syntax Ninja is mostly encoding agnostic, as long as the bytes Ninja cares about (like slashes in paths) are ASCII. This means e.g. UTF-8 or -ISO-8859-1 input files ought to work. +ISO-8859-1 input files ought to work. (To simplify some code, tabs +and carriage returns are currently disallowed; this could be fixed if +it really mattered to you.) Comments begin with `#` and extend to the end of the line. diff --git a/src/build_log.cc b/src/build_log.cc index 1b27be3..e72a93e 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -49,7 +49,7 @@ const int kCurrentVersion = 5; #define BIG_CONSTANT(x) (x##LLU) #endif // !defined(_MSC_VER) inline -uint64_t MurmurHash64A(const void* key, int len) { +uint64_t MurmurHash64A(const void* key, size_t len) { static const uint64_t seed = 0xDECAFBADDECAFBADull; const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); const int r = 47; @@ -58,11 +58,11 @@ uint64_t MurmurHash64A(const void* key, int len) { const uint64_t * end = data + (len/8); while(data != end) { uint64_t k = *data++; - k *= m; - k ^= k >> r; - k *= m; + k *= m; + k ^= k >> r; + k *= m; h ^= k; - h *= m; + h *= m; } const unsigned char* data2 = (const unsigned char*)data; switch(len & 7) @@ -80,7 +80,7 @@ uint64_t MurmurHash64A(const void* key, int len) { h *= m; h ^= h >> r; return h; -} +} #undef BIG_CONSTANT diff --git a/src/build_test.cc b/src/build_test.cc index 574ffb4..d4673ae 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -723,6 +723,31 @@ struct BuildWithLogTest : public BuildTest { BuildLog build_log_; }; +TEST_F(BuildWithLogTest, NotInLogButOnDisk) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n" +" command = cc\n" +"build out1: cc in\n")); + + // Create input/output that would be considered up to date when + // not considering the command line hash. + fs_.Create("in", now_, ""); + fs_.Create("out1", now_, ""); + string err; + + // Because it's not in the log, it should not be up-to-date until + // we build again. + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + commands_ran_.clear(); + state_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildWithLogTest, RestatTest) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule true\n" @@ -743,9 +768,22 @@ TEST_F(BuildWithLogTest, RestatTest) { fs_.Create("in", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out3 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + + fs_.Create("in", now_, ""); // "cc" touches out1, so we should build out2. But because "true" does not // touch out2, we should cancel the build of out3. - string err; EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); @@ -790,10 +828,24 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { fs_.Create("in", now_, ""); fs_.Create("out2", now_, ""); + // Do a pre-build so that there's commands in the log for the outputs, + // otherwise, the lack of an entry in the build log will cause out2 to rebuild + // regardless of restat. + string err; + EXPECT_TRUE(builder_.AddTarget("out2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + ASSERT_EQ("", err); + commands_ran_.clear(); + state_.Reset(); + + now_++; + fs_.Create("in", now_, ""); + fs_.Create("out2", now_, ""); + // Run a build, expect only the first command to run. // It doesn't touch its output (due to being the "true" command), so // we shouldn't run the dependent build. - string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index d0ed397..59bd18f 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -27,7 +27,7 @@ int main() { string err; char buf[200]; - int len = strlen(kPath); + size_t len = strlen(kPath); strcpy(buf, kPath); for (int j = 0; j < 5; ++j) { diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 03dad92..6887c91 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -149,7 +149,7 @@ yy4: yy5: { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -191,7 +191,7 @@ yy13: } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 68b6a95..1d4a177 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -70,7 +70,7 @@ bool DepfileParser::Parse(string* content, string* err) { } [a-zA-Z0-9+,/_:.~()@=-]+ { // Got a span of plain text. - int len = in - start; + int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. if (out < start) memmove(out, start, len); @@ -88,7 +88,7 @@ bool DepfileParser::Parse(string* content, string* err) { */ } - int len = out - filename; + int len = (int)(out - filename); const bool is_target = parsing_targets; if (len > 0 && filename[len - 1] == ':') { len--; // Strip off trailing colon, if any. diff --git a/src/graph.cc b/src/graph.cc index 18adeee..9654c1a 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -158,10 +158,15 @@ bool Edge::RecomputeOutputDirty(BuildLog* build_log, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - if (!rule_->generator() && build_log && - (entry || (entry = build_log->LookupByOutput(output->path())))) { - if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { - EXPLAIN("command line changed for %s", output->path().c_str()); + if (!rule_->generator() && build_log) { + if (entry || (entry = build_log->LookupByOutput(output->path()))) { + if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { + EXPLAIN("command line changed for %s", output->path().c_str()); + return true; + } + } + if (!entry) { + EXPLAIN("command line not found in log for %s", output->path().c_str()); return true; } } diff --git a/src/graph.h b/src/graph.h index 0ef4f3f..f487a22 100644 --- a/src/graph.h +++ b/src/graph.h @@ -199,12 +199,12 @@ struct Edge { // pointer...) int implicit_deps_; int order_only_deps_; - bool is_implicit(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_ - implicit_deps_ && + bool is_implicit(size_t index) { + return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && !is_order_only(index); } - bool is_order_only(int index) { - return index >= ((int)inputs_.size()) - order_only_deps_; + bool is_order_only(size_t index) { + return index >= inputs_.size() - order_only_deps_; } bool is_phony() const; diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index 6736109..d0eabde 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -14,6 +14,11 @@ #include "build_log.h" +#include <algorithm> +using namespace std; + +#include <time.h> + int random(int low, int high) { return int(low + (rand() / double(RAND_MAX)) * (high - low) + 0.5); } @@ -22,7 +27,7 @@ void RandomCommand(char** s) { int len = random(5, 100); *s = new char[len]; for (int i = 0; i < len; ++i) - (*s)[i] = random(32, 127); + (*s)[i] = (char)random(32, 127); } int main() { @@ -32,7 +37,7 @@ int main() { char** commands = new char*[N]; pair<uint64_t, int>* hashes = new pair<uint64_t, int>[N]; - srand(time(NULL)); + srand((int)time(NULL)); for (int i = 0; i < N; ++i) { RandomCommand(&commands[i]); diff --git a/src/hash_map.h b/src/hash_map.h index 88c2681..9904fb8 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -19,7 +19,7 @@ // MurmurHash2, by Austin Appleby static inline -unsigned int MurmurHash2(const void* key, int len) { +unsigned int MurmurHash2(const void* key, size_t len) { static const unsigned int seed = 0xDECAFBAD; const unsigned int m = 0x5bd1e995; const int r = 24; diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc new file mode 100644 index 0000000..134cfb8 --- /dev/null +++ b/src/includes_normalize-win32.cc @@ -0,0 +1,115 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include "string_piece.h" +#include "util.h" + +#include <algorithm> +#include <iterator> +#include <sstream> + +#include <windows.h> + +namespace { + +/// Return true if paths a and b are on the same Windows drive. +bool SameDrive(StringPiece a, StringPiece b) { + char a_absolute[_MAX_PATH]; + char b_absolute[_MAX_PATH]; + GetFullPathName(a.AsString().c_str(), sizeof(a_absolute), a_absolute, NULL); + GetFullPathName(b.AsString().c_str(), sizeof(b_absolute), b_absolute, NULL); + char a_drive[_MAX_DIR]; + char b_drive[_MAX_DIR]; + _splitpath(a_absolute, a_drive, NULL, NULL, NULL); + _splitpath(b_absolute, b_drive, NULL, NULL, NULL); + return _stricmp(a_drive, b_drive) == 0; +} + +} // anonymous namespace + +string IncludesNormalize::Join(const vector<string>& list, char sep) { + string ret; + for (size_t i = 0; i < list.size(); ++i) { + ret += list[i]; + if (i != list.size() - 1) + ret += sep; + } + return ret; +} + +vector<string> IncludesNormalize::Split(const string& input, char sep) { + vector<string> elems; + stringstream ss(input); + string item; + while (getline(ss, item, sep)) + elems.push_back(item); + return elems; +} + +string IncludesNormalize::ToLower(const string& s) { + string ret; + transform(s.begin(), s.end(), back_inserter(ret), tolower); + return ret; +} + +string IncludesNormalize::AbsPath(StringPiece s) { + char result[_MAX_PATH]; + GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + return result; +} + +string IncludesNormalize::Relativize(StringPiece path, const string& start) { + vector<string> start_list = Split(AbsPath(start), '\\'); + vector<string> path_list = Split(AbsPath(path), '\\'); + int i; + for (i = 0; i < static_cast<int>(min(start_list.size(), path_list.size())); + ++i) { + if (ToLower(start_list[i]) != ToLower(path_list[i])) + break; + } + + vector<string> rel_list; + for (int j = 0; j < static_cast<int>(start_list.size() - i); ++j) + rel_list.push_back(".."); + for (int j = i; j < static_cast<int>(path_list.size()); ++j) + rel_list.push_back(path_list[j]); + if (rel_list.size() == 0) + return "."; + return Join(rel_list, '\\'); +} + +string IncludesNormalize::Normalize(const string& input, + const char* relative_to) { + char copy[_MAX_PATH]; + size_t len = input.size(); + strncpy(copy, input.c_str(), input.size() + 1); + for (size_t j = 0; j < len; ++j) + if (copy[j] == '/') + copy[j] = '\\'; + string err; + if (!CanonicalizePath(copy, &len, &err)) { + Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); + } + string curdir; + if (!relative_to) { + curdir = AbsPath("."); + relative_to = curdir.c_str(); + } + StringPiece partially_fixed(copy, len); + if (!SameDrive(partially_fixed, relative_to)) + return ToLower(partially_fixed.AsString()); + return ToLower(Relativize(partially_fixed, relative_to)); +} diff --git a/src/includes_normalize.h b/src/includes_normalize.h new file mode 100644 index 0000000..43527af --- /dev/null +++ b/src/includes_normalize.h @@ -0,0 +1,35 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> +#include <vector> +using namespace std; + +struct StringPiece; + +/// Utility functions for normalizing include paths on Windows. +/// TODO: this likely duplicates functionality of CanonicalizePath; refactor. +struct IncludesNormalize { + // Internal utilities made available for testing, maybe useful otherwise. + static string Join(const vector<string>& list, char sep); + static vector<string> Split(const string& input, char sep); + static string ToLower(const string& s); + static string AbsPath(StringPiece s); + static string Relativize(StringPiece path, const string& start); + + /// Normalize by fixing slashes style, fixing redundant .. and . and makes the + /// path relative to |relative_to|. Case is normalized to lowercase on + /// Windows too. + static string Normalize(const string& input, const char* relative_to); +}; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc new file mode 100644 index 0000000..77b5b3b --- /dev/null +++ b/src/includes_normalize_test.cc @@ -0,0 +1,98 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "includes_normalize.h" + +#include <gtest/gtest.h> + +#include "test.h" +#include "util.h" + +TEST(IncludesNormalize, Simple) { + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); + EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); +} + +namespace { + +string GetCurDir() { + char buf[_MAX_PATH]; + _getcwd(buf, sizeof(buf)); + vector<string> parts = IncludesNormalize::Split(string(buf), '\\'); + return parts[parts.size() - 1]; +} + +} // namespace + +TEST(IncludesNormalize, WithRelative) { + string currentdir = IncludesNormalize::ToLower(GetCurDir()); + EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); + EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL)); + EXPECT_EQ(string("..\\") + currentdir + string("\\a"), + IncludesNormalize::Normalize("a", "../b")); + EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), + IncludesNormalize::Normalize("a/b", "../c")); + EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); + EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); +} + +TEST(IncludesNormalize, Case) { + EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); + EXPECT_EQ("bdef", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./b", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\.\\B", NULL)); + EXPECT_EQ("a\\b", IncludesNormalize::Normalize("A\\./B", NULL)); +} + +TEST(IncludesNormalize, Join) { + vector<string> x; + EXPECT_EQ("", IncludesNormalize::Join(x, ':')); + x.push_back("alpha"); + EXPECT_EQ("alpha", IncludesNormalize::Join(x, ':')); + x.push_back("beta"); + x.push_back("gamma"); + EXPECT_EQ("alpha:beta:gamma", IncludesNormalize::Join(x, ':')); +} + +TEST(IncludesNormalize, Split) { + EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':')); + EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':')); + EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':')); +} + +TEST(IncludesNormalize, ToLower) { + EXPECT_EQ("", IncludesNormalize::ToLower("")); + EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff")); + EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS")); + EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS")); +} + +TEST(IncludesNormalize, DifferentDrive) { + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "c:\\vs08")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:\\vs08\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("p:\\vs08\\stuff.h", + IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); + // TODO: this fails; fix it. + //EXPECT_EQ("P:\\wee\\stuff.h", + // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); +} diff --git a/src/lexer.cc b/src/lexer.cc index f4036d4..5d7d185 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -30,7 +30,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); diff --git a/src/lexer.in.cc b/src/lexer.in.cc index ec3ad6b..7ae9c61 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -29,7 +29,7 @@ bool Lexer::Error(const string& message, string* err) { context = p + 1; } } - int col = last_token_ ? last_token_ - context : 0; + int col = last_token_ ? (int)(last_token_ - context) : 0; char buf[1024]; snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line); diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc new file mode 100644 index 0000000..8e440fe --- /dev/null +++ b/src/msvc_helper-win32.cc @@ -0,0 +1,164 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <string.h> +#include <windows.h> + +#include "includes_normalize.h" +#include "util.h" + +namespace { + +/// Return true if \a input ends with \a needle. +bool EndsWith(const string& input, const string& needle) { + return (input.size() >= needle.size() && + input.substr(input.size() - needle.size()) == needle); +} + +} // anonymous namespace + +// static +string CLWrapper::FilterShowIncludes(const string& line) { + static const char kMagicPrefix[] = "Note: including file: "; + const char* in = line.c_str(); + const char* end = in + line.size(); + + if (end - in > (int)sizeof(kMagicPrefix) - 1 && + memcmp(in, kMagicPrefix, sizeof(kMagicPrefix) - 1) == 0) { + in += sizeof(kMagicPrefix) - 1; + while (*in == ' ') + ++in; + return line.substr(in - line.c_str()); + } + return ""; +} + +// static +bool CLWrapper::IsSystemInclude(const string& path) { + // TODO: this is a heuristic, perhaps there's a better way? + return (path.find("program files") != string::npos || + path.find("microsoft visual studio") != string::npos); +} + +// static +bool CLWrapper::FilterInputFilename(const string& line) { + // TODO: other extensions, like .asm? + return EndsWith(line, ".c") || + EndsWith(line, ".cc") || + EndsWith(line, ".cxx") || + EndsWith(line, ".cpp"); +} + +int CLWrapper::Run(const string& command, string* extra_output) { + SECURITY_ATTRIBUTES security_attributes = {}; + security_attributes.nLength = sizeof(SECURITY_ATTRIBUTES); + security_attributes.bInheritHandle = TRUE; + + // Must be inheritable so subprocesses can dup to children. + HANDLE nul = CreateFile("NUL", GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | + FILE_SHARE_DELETE, + &security_attributes, OPEN_EXISTING, 0, NULL); + if (nul == INVALID_HANDLE_VALUE) + Fatal("couldn't open nul"); + + HANDLE stdout_read, stdout_write; + if (!CreatePipe(&stdout_read, &stdout_write, &security_attributes, 0)) + Win32Fatal("CreatePipe"); + + if (!SetHandleInformation(stdout_read, HANDLE_FLAG_INHERIT, 0)) + Win32Fatal("SetHandleInformation"); + + PROCESS_INFORMATION process_info = {}; + STARTUPINFO startup_info = {}; + startup_info.cb = sizeof(STARTUPINFO); + startup_info.hStdInput = nul; + startup_info.hStdError = stdout_write; + startup_info.hStdOutput = stdout_write; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + if (!CreateProcessA(NULL, (char*)command.c_str(), NULL, NULL, + /* inherit handles */ TRUE, 0, + env_block_, NULL, + &startup_info, &process_info)) { + Win32Fatal("CreateProcess"); + } + + if (!CloseHandle(nul) || + !CloseHandle(stdout_write)) { + Win32Fatal("CloseHandle"); + } + + // Read output of the subprocess and parse it. + string output; + DWORD read_len = 1; + while (read_len) { + char buf[64 << 10]; + read_len = 0; + if (!::ReadFile(stdout_read, buf, sizeof(buf), &read_len, NULL) && + GetLastError() != ERROR_BROKEN_PIPE) { + Win32Fatal("ReadFile"); + } + output.append(buf, read_len); + + // Loop over all lines in the output and process them. + for (;;) { + size_t ofs = output.find_first_of("\r\n"); + if (ofs == string::npos) + break; + string line = output.substr(0, ofs); + + string include = FilterShowIncludes(line); + if (!include.empty()) { + include = IncludesNormalize::Normalize(include, NULL); + if (!IsSystemInclude(include)) + includes_.push_back(include); + } else if (FilterInputFilename(line)) { + // Drop it. + // TODO: if we support compiling multiple output files in a single + // cl.exe invocation, we should stash the filename. + } else { + if (extra_output) { + extra_output->append(line); + extra_output->append("\n"); + } else { + printf("%s\n", line.c_str()); + } + } + + if (ofs < output.size() && output[ofs] == '\r') + ++ofs; + if (ofs < output.size() && output[ofs] == '\n') + ++ofs; + output = output.substr(ofs); + } + } + + if (WaitForSingleObject(process_info.hProcess, INFINITE) == WAIT_FAILED) + Win32Fatal("WaitForSingleObject"); + + DWORD exit_code = 0; + if (!GetExitCodeProcess(process_info.hProcess, &exit_code)) + Win32Fatal("GetExitCodeProcess"); + + if (!CloseHandle(stdout_read) || + !CloseHandle(process_info.hProcess) || + !CloseHandle(process_info.hThread)) { + Win32Fatal("CloseHandle"); + } + + return exit_code; +} diff --git a/src/msvc_helper.h b/src/msvc_helper.h new file mode 100644 index 0000000..f623520 --- /dev/null +++ b/src/msvc_helper.h @@ -0,0 +1,54 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> +#include <vector> +using namespace std; + +/// Visual Studio's cl.exe requires some massaging to work with Ninja; +/// for example, it emits include information on stderr in a funny +/// format when building with /showIncludes. This class wraps a CL +/// process and parses that output to extract the file list. +struct CLWrapper { + CLWrapper() : env_block_(NULL) {} + + /// Set the environment block (as suitable for CreateProcess) to be used + /// by Run(). + void SetEnvBlock(void* env_block) { env_block_ = env_block; } + + /// Start a process and parse its output. Returns its exit code. + /// Any non-parsed output is buffered into \a extra_output if provided, + /// otherwise it is printed to stdout while the process runs. + /// Crashes (calls Fatal()) on error. + int Run(const string& command, string* extra_output=NULL); + + /// Parse a line of cl.exe output and extract /showIncludes info. + /// If a dependency is extracted, returns a nonempty string. + /// Exposed for testing. + static string FilterShowIncludes(const string& line); + + /// Return true if a mentioned include file is a system path. + /// Expects the path to already by normalized (including lower case). + /// Filtering these out reduces dependency information considerably. + static bool IsSystemInclude(const string& path); + + /// Parse a line of cl.exe output and return true if it looks like + /// it's printing an input filename. This is a heuristic but it appears + /// to be the best we can do. + /// Exposed for testing. + static bool FilterInputFilename(const string& line); + + void* env_block_; + vector<string> includes_; +}; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc new file mode 100644 index 0000000..f265010 --- /dev/null +++ b/src/msvc_helper_main-win32.cc @@ -0,0 +1,115 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <windows.h> + +#include "util.h" + +#include "getopt.h" + +namespace { + +void Usage() { + printf( +"ninja-msvc-helper: adjust msvc command-line tools for use by ninja.\n" +"\n" +"usage: ninja-mvsc-helper [options] -- command args\n" +"options:\n" +" -e ENVFILE load environment block from ENVFILE as environment\n" +" -r BASE normalize paths and make relative to BASE before output\n" +" -o FILE write output dependency information to FILE.d\n" + ); +} + +void PushPathIntoEnvironment(const string& env_block) { + const char* as_str = env_block.c_str(); + while (as_str[0]) { + if (_strnicmp(as_str, "path=", 5) == 0) { + _putenv(as_str); + return; + } else { + as_str = &as_str[strlen(as_str) + 1]; + } + } +} + +} // anonymous namespace + +int main(int argc, char** argv) { + const char* output_filename = NULL; + const char* relative_to = NULL; + const char* envfile = NULL; + + const option kLongOptions[] = { + { "help", no_argument, NULL, 'h' }, + { NULL, 0, NULL, 0 } + }; + int opt; + while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'e': + envfile = optarg; + break; + case 'o': + output_filename = optarg; + break; + case 'r': + relative_to = optarg; + break; + case 'h': + default: + Usage(); + return 0; + } + } + + if (!output_filename) + Fatal("-o required"); + + string env; + if (envfile) { + string err; + if (ReadFile(envfile, &env, &err) != 0) + Fatal("couldn't open %s: %s", envfile, err.c_str()); + PushPathIntoEnvironment(env); + } + + char* command = GetCommandLine(); + command = strstr(command, " -- "); + if (!command) { + Fatal("expected command line to end with \" -- command args\""); + } + command += 4; + + CLWrapper cl; + if (!env.empty()) + cl.SetEnvBlock((void*)env.data()); + int exit_code = cl.Run(command); + + string depfile = string(output_filename) + ".d"; + FILE* output = fopen(depfile.c_str(), "w"); + if (!output) { + Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str()); + } + fprintf(output, "%s: ", output_filename); + for (vector<string>::iterator i = cl.includes_.begin(); + i != cl.includes_.end(); ++i) { + fprintf(output, "%s\n", i->c_str()); + } + fclose(output); + + return exit_code; +} diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc new file mode 100644 index 0000000..29fefd4 --- /dev/null +++ b/src/msvc_helper_test.cc @@ -0,0 +1,83 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "msvc_helper.h" + +#include <gtest/gtest.h> + +#include "test.h" +#include "util.h" + +TEST(MSVCHelperTest, ShowIncludes) { + ASSERT_EQ("", CLWrapper::FilterShowIncludes("")); + + ASSERT_EQ("", CLWrapper::FilterShowIncludes("Sample compiler output")); + ASSERT_EQ("c:\\Some Files\\foobar.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\Some Files\\foobar.h")); + ASSERT_EQ("c:\\initspaces.h", + CLWrapper::FilterShowIncludes("Note: including file: " + "c:\\initspaces.h")); +} + +TEST(MSVCHelperTest, FilterInputFilename) { + ASSERT_TRUE(CLWrapper::FilterInputFilename("foobar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("foo bar.cc")); + ASSERT_TRUE(CLWrapper::FilterInputFilename("baz.c")); + + ASSERT_FALSE(CLWrapper::FilterInputFilename( + "src\\cl_helper.cc(166) : fatal error C1075: end " + "of file found ...")); +} + +TEST(MSVCHelperTest, Run) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo&& echo Note: including file: foo.h&&echo bar\"", + &output); + ASSERT_EQ("foo\nbar\n", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("foo.h", cl.includes_[0]); +} + +TEST(MSVCHelperTest, RunFilenameFilter) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo foo.cc&& echo cl: warning\"", + &output); + ASSERT_EQ("cl: warning\n", output); +} + +TEST(MSVCHelperTest, RunSystemInclude) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo Note: including file: c:\\Program Files\\foo.h&&" + "echo Note: including file: d:\\Microsoft Visual Studio\\bar.h&&" + "echo Note: including file: path.h\"", + &output); + // We should have dropped the first two includes because they look like + // system headers. + ASSERT_EQ("", output); + ASSERT_EQ(1u, cl.includes_.size()); + ASSERT_EQ("path.h", cl.includes_[0]); +} + +TEST(MSVCHelperTest, EnvBlock) { + char env_block[] = "foo=bar\0"; + CLWrapper cl; + cl.SetEnvBlock(env_block); + string output; + cl.Run("cmd /c \"echo foo is %foo%", &output); + ASSERT_EQ("foo is bar\n", output); +} diff --git a/src/ninja.cc b/src/ninja.cc index b645b90..00bd104 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -77,14 +77,12 @@ void Usage(const BuildConfig& config) { "usage: ninja [options] [targets...]\n" "\n" "if targets are unspecified, builds the 'default' target (see manual).\n" -"targets are paths, with additional special syntax:\n" -" 'target^' means 'the first output that uses target'.\n" -" example: 'ninja foo.cc^' will likely build foo.o.\n" "\n" "options:\n" +" --version print ninja version (\"%s\")\n" +"\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" -" -V print ninja version (\"%s\")\n" "\n" " -j N run N jobs in parallel [default=%d]\n" " -l N do not start new jobs if the load average is greater than N\n" @@ -92,13 +90,12 @@ void Usage(const BuildConfig& config) { " (not yet implemented on Windows)\n" #endif " -k N keep going until N jobs fail [default=1]\n" -" -n dry run (don't run commands but pretend they succeeded)\n" +" -n dry run (don't run commands but act like they succeeded)\n" " -v show all command lines while building\n" "\n" " -d MODE enable debugging (use -d list to list modes)\n" -" -t TOOL run a subtool\n" -" use '-t list' to list subtools.\n" -" terminates toplevel options; further flags are passed to the tool.\n", +" -t TOOL run a subtool (use -t list to list subtools)\n" +" terminates toplevel options; further flags are passed to the tool\n", kVersion, config.parallelism); } @@ -649,8 +646,10 @@ int NinjaMain(int argc, char** argv) { globals.config.parallelism = GuessParallelism(); + enum { OPT_VERSION = 1 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, OPT_VERSION }, { NULL, 0, NULL, 0 } }; @@ -701,7 +700,7 @@ int NinjaMain(int argc, char** argv) { case 'C': working_dir = optarg; break; - case 'V': + case OPT_VERSION: printf("%s\n", kVersion); return 0; case 'h': diff --git a/src/string_piece.h b/src/string_piece.h index ad1153e..b1bf105 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -31,7 +31,7 @@ struct StringPiece { StringPiece(const string& str) : str_(str.data()), len_(str.size()) {} StringPiece(const char* str) : str_(str), len_(strlen(str)) {} - StringPiece(const char* str, int len) : str_(str), len_(len) {} + StringPiece(const char* str, size_t len) : str_(str), len_(len) {} bool operator==(const StringPiece& other) const { return len_ == other.len_ && memcmp(str_, other.str_, len_) == 0; @@ -47,7 +47,7 @@ struct StringPiece { } const char* str_; - int len_; + size_t len_; }; #endif // NINJA_STRINGPIECE_H_ diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 38b6957..4b103a5 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -20,14 +20,6 @@ #include "util.h" -namespace { - -void Win32Fatal(const char* function) { - Fatal("%s: %s", function, GetLastErrorString().c_str()); -} - -} // anonymous namespace - Subprocess::Subprocess() : child_(NULL) , overlapped_(), is_reading_(false) { } diff --git a/src/util.cc b/src/util.cc index 4bd34e6..580f22a 100644 --- a/src/util.cc +++ b/src/util.cc @@ -81,7 +81,7 @@ void Error(const char* msg, ...) { bool CanonicalizePath(string* path, string* err) { METRIC_RECORD("canonicalize str"); - int len = path->size(); + size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; @@ -91,7 +91,7 @@ bool CanonicalizePath(string* path, string* err) { return true; } -bool CanonicalizePath(char* path, int* len, string* err) { +bool CanonicalizePath(char* path, size_t* len, string* err) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); @@ -258,6 +258,10 @@ string GetLastErrorString() { LocalFree(msg_buf); return msg; } + +void Win32Fatal(const char* function) { + Fatal("%s: %s", function, GetLastErrorString().c_str()); +} #endif static bool islatinalpha(int c) { @@ -334,7 +338,7 @@ string ElideMiddle(const string& str, size_t width) { const int kMargin = 3; // Space for "...". string result = str; if (result.size() + kMargin > width) { - int elide_size = (width - kMargin) / 2; + size_t elide_size = (width - kMargin) / 2; result = result.substr(0, elide_size) + "..." + result.substr(result.size() - elide_size, elide_size); @@ -39,9 +39,10 @@ void Error(const char* msg, ...); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". bool CanonicalizePath(string* path, string* err); -bool CanonicalizePath(char* path, int* len, string* err); +bool CanonicalizePath(char* path, size_t* len, string* err); -/// Read a file to a string. +/// Read a file to a string (in text mode: with CRLF conversion +/// on Windows). /// Returns -errno and fills in \a err on error. int ReadFile(const string& path, string* contents, string* err); @@ -81,6 +82,9 @@ string ElideMiddle(const string& str, size_t width); #ifdef _WIN32 /// Convert the value returned by GetLastError() into a string. string GetLastErrorString(); + +/// Calls Fatal() with a function name and GetLastErrorString. +void Win32Fatal(const char* function); #endif #endif // NINJA_UTIL_H_ diff --git a/src/util_test.cc b/src/util_test.cc index 17e2704..4776546 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -121,18 +121,18 @@ TEST(CanonicalizePath, AbsolutePath) { TEST(CanonicalizePath, NotNullTerminated) { string path; string err; - int len; + size_t len; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("foo"), static_cast<size_t>(len)); + EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); - EXPECT_EQ(strlen("file"), static_cast<size_t>(len)); + EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } |