diff options
96 files changed, 3177 insertions, 1425 deletions
@@ -1,4 +1,5 @@ *.pyc +*.obj *.exe *.pdb *.ilk diff --git a/.travis.yml b/.travis.yml index d7bee6f..544db6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,4 @@ language: cpp compiler: - gcc - clang -before_install: - - sudo apt-get update -qq - - sudo apt-get install libgtest-dev -script: ./bootstrap.py && ./configure.py --with-gtest=/usr/src/gtest && ./ninja ninja_test && ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots +script: ./configure.py --bootstrap && ./ninja ninja_test && ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots && ./misc/ninja_syntax_test.py @@ -2,14 +2,25 @@ `./configure.py` generates the `build.ninja` files used to build ninja. It accepts various flags to adjust build parameters. +Run './configure.py --help' for more configuration options. The primary build target of interest is `ninja`, but when hacking on -Ninja your changes should be testable so it's more useful to build -and run `ninja_test` when developing. +Ninja your changes should be testable so it's more useful to build and +run `ninja_test` when developing. -(`./bootstrap.py` creates a bootstrap `ninja` and runs the above -process; it's only necessary to run if you don't have a copy of -`ninja` to build with.) +### Bootstrapping + +Ninja is built using itself. To bootstrap the first binary, run the +configure script as `./configure.py --bootstrap`. This first compiles +all non-test source files together, then re-builds Ninja using itself. +You should end up with a `ninja` binary (or `ninja.exe`) in the source root. + +#### Windows + +On Windows, you'll need to install Python to run `configure.py`, and +run everything under a Visual Studio Tools Command Prompt (or after +running `vcvarsall` in a normal command prompt). See below if you +want to use mingw or some other compiler instead using Visual Studio. ### Adjusting build flags @@ -42,7 +53,7 @@ obvious exceptions (fixing typos in comments don't need tests). I am very wary of changes that increase the complexity of Ninja (in particular, new build file syntax or command-line flags) or increase -the maintenance burden of Ninja. Ninja is already successfully in use +the maintenance burden of Ninja. Ninja is already successfully used by hundreds of developers for large projects and it already achieves (most of) the goals I set out for it to do. It's probably best to discuss new feature ideas on the mailing list before I shoot down your @@ -50,26 +61,6 @@ patch. ## Testing -### Installing gtest - -The `ninja_test` binary, containing all the tests, depends on the -googletest (gtest) library. - -* On older Ubuntus it'll install as libraries into `/usr/lib`: - - apt-get install libgtest - -* On newer Ubuntus it's only distributed as source - - apt-get install libgtest-dev - ./configure.py --with-gtest=/usr/src/gtest - -* Otherwise you need to download it, unpack it, and pass - `--with-gtest` to `configure.py`. Get it from [its downloads - page](http://code.google.com/p/googletest/downloads/list); [this - direct download link might work - too](http://googletest.googlecode.com/files/gtest-1.6.0.zip). - ### Test-driven development Set your build command to @@ -83,7 +74,7 @@ build "all" before committing to verify the other source still works! ## Testing performance impact of changes If you have a Chrome build handy, it's a good test case. Otherwise, -[the github downoads page](https://github.com/martine/ninja/downloads) +[the github downoads page](https://github.com/ninja-build/ninja/releases) has a copy of the Chrome build files (and depfiles). You can untar that, then run @@ -146,14 +137,15 @@ it's locked while in use. * Install Visual Studio (Express is fine), [Python for Windows][], and (if making changes) googletest (see above instructions) -* In a Visual Studio command prompt: `python bootstrap.py` +* In a Visual Studio command prompt: `python configure.py --bootstrap` [Python for Windows]: http://www.python.org/getit/windows/ ### Via mingw on Windows (not well supported) * Install mingw, msys, and python -* In the mingw shell, put Python in your path, and `python bootstrap.py` +* In the mingw shell, put Python in your path, and + `python configure.py --bootstrap` * To reconfigure, run `python configure.py` * Remember to strip the resulting executable if size matters to you @@ -167,6 +159,12 @@ Setup on Ubuntu Precise: * `sudo apt-get install gcc-mingw-w64-i686 g++-mingw-w64-i686 wine` * `export CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ AR=i686-w64-mingw32-ar` +Setup on Arch: +* Uncomment the `[multilib]` section of `/etc/pacman.conf` and `sudo pacman -Sy`. +* `sudo pacman -S mingw-w64-gcc wine` +* `export CC=x86_64-w64-mingw32-cc CXX=x86_64-w64-mingw32-c++ AR=x86_64-w64-mingw32-ar` +* `export CFLAGS=-I/usr/x86_64-w64-mingw32/include` + Then run: * `./configure.py --platform=mingw --host=linux` * Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` @@ -178,3 +176,49 @@ The trick is to install just the compilers, and not all of Visual Studio, by following [these instructions][win7sdk]. [win7sdk]: http://www.kegel.com/wine/cl-howto-win7sdk.html + +### Using gcov + +Do a clean debug build with the right flags: + + CFLAGS=-coverage LDFLAGS=-coverage ./configure.py --debug + ninja -t clean ninja_test && ninja ninja_test + +Run the test binary to generate `.gcda` and `.gcno` files in the build +directory, then run gcov on the .o files to generate `.gcov` files in the +root directory: + + ./ninja_test + gcov build/*.o + +Look at the generated `.gcov` files directly, or use your favorite gcov viewer. + +### Using afl-fuzz + +Build with afl-clang++: + + CXX=path/to/afl-1.20b/afl-clang++ ./configure.py + ninja + +Then run afl-fuzz like so: + + afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ + +You can pass `-x misc/afl-fuzz-tokens` to use the token dictionary. In my +testing, that did not seem more effective though. + +#### Using afl-fuzz with asan + +If you want to use asan (the `isysroot` bit is only needed on OS X; if clang +can't find C++ standard headers make sure your LLVM checkout includes a libc++ +checkout and has libc++ installed in the build directory): + + CFLAGS="-fsanitize=address -isysroot $(xcrun -show-sdk-path)" \ + LDFLAGS=-fsanitize=address CXX=path/to/afl-1.20b/afl-clang++ \ + ./configure.py + AFL_CXX=path/to/clang++ ninja + +Make sure ninja can find the asan runtime: + + DYLD_LIBRARY_PATH=path/to//lib/clang/3.7.0/lib/darwin/ \ + afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ @@ -1,16 +1,21 @@ Ninja is a small build system with a focus on speed. -http://martine.github.com/ninja/ +http://ninja-build.org/ -See the manual -- http://martine.github.com/ninja/manual.html or +See the manual -- http://ninja-build.org/manual.html or doc/manual.asciidoc included in the distribution -- for background and more details. -To build, run ./bootstrap.py. It first blindly compiles all non-test -source files together, then re-builds Ninja using itself. You should -end up with a 'ninja' binary in the source root. Run './ninja -h' for -help. +Binaries for Linux, Mac, and Windows are available at + https://github.com/ninja-build/ninja/releases +Run './ninja -h' for Ninja help. -There is no installation step. The only file of interest to a user -is the resulting ninja binary. +To build your own binary, on many platforms it should be sufficient to +just run `./configure.py --bootstrap`; for more details see HACKING.md. +(Also read that before making changes to Ninja, as it has advice.) + +Installation is not necessary because the only required file is the +resulting ninja binary. However, to enable features like Bash +completion and Emacs and Vim editing modes, some files in misc/ must be +copied to appropriate locations. If you're interested in making changes to Ninja, read HACKING.md first. @@ -1,22 +1,31 @@ Notes to myself on all the steps to make for a Ninja release. Push new release branch: -1. update src/version.cc with new version (with ".git") -2. git checkout release; git merge master -3. fix version number in src/version.cc (it will likely conflict in the above) -4. fix version in doc/manual.asciidoc -5. commit, tag, push (don't forget to push --tags) -6. construct release notes from prior notes +1. Consider sending a heads-up to the ninja-build mailing list first +2. update src/version.cc with new version (with ".git"), then + git commit -a -m 'mark this 1.5.0.git' +3. git checkout release; git merge master +4. fix version number in src/version.cc (it will likely conflict in the above) +5. fix version in doc/manual.asciidoc +6. commit, tag, push (don't forget to push --tags) + git commit -a -m v1.5.0; git push origin release + git tag v1.5.0; git push --tags + # Push the 1.5.0.git change on master too: + git checkout master; git push origin master +7. construct release notes from prior notes credits: git shortlog -s --no-merges REV.. Release on github: -1. (haven't tried this yet) - https://github.com/blog/1547-release-your-software +1. https://github.com/blog/1547-release-your-software + Add binaries to https://github.com/ninja-build/ninja/releases Make announcement on mailing list: 1. copy old mail Update website: -(note to self: website is now in github.com/martine/martine.github.io) -1. rebuild manual, put in place on website -2. update home page mention of latest version. +1. Make sure your ninja checkout is on the v1.5.0 tag +2. Clone https://github.com/ninja-build/ninja-build.github.io +3. In that repo, `cd ninja && ./update-docs.sh` +4. Update index.html with newest version and link to release notes +5. git commit -m 'run update-docs.sh, 1.5.0 release' +6. git push origin master diff --git a/bootstrap.py b/bootstrap.py index 026396b..56eab64 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -15,152 +15,9 @@ from __future__ import print_function -from optparse import OptionParser -import sys -import os -import glob -import errno -import shlex -import shutil import subprocess -import platform_helper - -os.chdir(os.path.dirname(os.path.abspath(__file__))) - -parser = OptionParser() - -parser.add_option('--verbose', action='store_true', - help='enable verbose build',) -parser.add_option('--x64', action='store_true', - help='force 64-bit build (Windows)',) -parser.add_option('--platform', - help='target platform (' + - '/'.join(platform_helper.platforms()) + ')', - choices=platform_helper.platforms()) -parser.add_option('--force-pselect', action='store_true', - help='ppoll() is used by default where available, ' - 'but some platforms might need to use pselect instead',) -(options, conf_args) = parser.parse_args() - - -platform = platform_helper.Platform(options.platform) -conf_args.append("--platform=" + platform.platform()) - -def run(*args, **kwargs): - returncode = subprocess.call(*args, **kwargs) - if returncode != 0: - sys.exit(returncode) - -# Compute system-specific CFLAGS/LDFLAGS as used in both in the below -# g++ call as well as in the later configure.py. -cflags = os.environ.get('CFLAGS', '').split() -ldflags = os.environ.get('LDFLAGS', '').split() -if platform.is_freebsd() or platform.is_openbsd() or platform.is_bitrig(): - cflags.append('-I/usr/local/include') - ldflags.append('-L/usr/local/lib') - -print('Building ninja manually...') - -try: - os.mkdir('build') -except OSError: - e = sys.exc_info()[1] - if e.errno != errno.EEXIST: - raise - -sources = [] -for src in glob.glob('src/*.cc'): - if src.endswith('test.cc') or src.endswith('.in.cc'): - continue - if src.endswith('bench.cc'): - continue - - filename = os.path.basename(src) - if filename == 'browse.cc': # Depends on generated header. - continue - - if platform.is_windows(): - if src.endswith('-posix.cc'): - continue - else: - if src.endswith('-win32.cc'): - continue - - sources.append(src) - -if platform.is_windows(): - sources.append('src/getopt.c') - -if platform.is_msvc(): - cl = 'cl' - vcdir = os.environ.get('VCINSTALLDIR') - if vcdir: - if options.x64: - cl = os.path.join(vcdir, 'bin', 'x86_amd64', 'cl.exe') - if not os.path.exists(cl): - cl = os.path.join(vcdir, 'bin', 'amd64', 'cl.exe') - else: - cl = os.path.join(vcdir, 'bin', 'cl.exe') - args = [cl, '/nologo', '/EHsc', '/DNOMINMAX'] -else: - args = shlex.split(os.environ.get('CXX', 'g++')) - cflags.extend(['-Wno-deprecated', - '-DNINJA_PYTHON="' + sys.executable + '"', - '-DNINJA_BOOTSTRAP']) - if platform.is_windows(): - cflags.append('-D_WIN32_WINNT=0x0501') - if options.x64: - cflags.append('-m64') -if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and \ - not options.force_pselect: - cflags.append('-DUSE_PPOLL') -if options.force_pselect: - conf_args.append("--force-pselect") -args.extend(cflags) -args.extend(ldflags) -binary = 'ninja.bootstrap' -if platform.is_windows(): - binary = 'ninja.bootstrap.exe' -args.extend(sources) -if platform.is_msvc(): - args.extend(['/link', '/out:' + binary]) -else: - args.extend(['-o', binary]) - -if options.verbose: - print(' '.join(args)) - -try: - run(args) -except: - print('Failure running:', args) - raise - -verbose = [] -if options.verbose: - verbose = ['-v'] - -if platform.is_windows(): - print('Building ninja using itself...') - run([sys.executable, 'configure.py'] + conf_args) - run(['./' + binary] + verbose) - - # Copy the new executable over the bootstrap one. - shutil.copyfile('ninja.exe', binary) - - # Clean up. - for obj in glob.glob('*.obj'): - os.unlink(obj) - - print(""" -Done! +import sys -Note: to work around Windows file locking, where you can't rebuild an -in-use binary, to run ninja after making any changes to build ninja -itself you should run ninja.bootstrap instead.""") -else: - print('Building ninja using itself...') - run([sys.executable, 'configure.py'] + conf_args) - run(['./' + binary] + verbose) - os.unlink(binary) - print('Done!') +print('DEPRECATED: this script will be deleted.') +print('use "configure.py --bootstrap" instead.') +subprocess.check_call([sys.executable, 'configure.py', '--bootstrap']) diff --git a/configure.py b/configure.py index c5a6abd..0cff196 100755 --- a/configure.py +++ b/configure.py @@ -23,29 +23,180 @@ from __future__ import print_function from optparse import OptionParser import os +import pipes +import string +import subprocess import sys -import platform_helper -sys.path.insert(0, 'misc') +sourcedir = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(sourcedir, 'misc')) import ninja_syntax + +class Platform(object): + """Represents a host/target platform and its specific build attributes.""" + def __init__(self, platform): + self._platform = platform + if self._platform is not None: + return + self._platform = sys.platform + if self._platform.startswith('linux'): + self._platform = 'linux' + elif self._platform.startswith('freebsd'): + self._platform = 'freebsd' + elif self._platform.startswith('gnukfreebsd'): + self._platform = 'freebsd' + elif self._platform.startswith('openbsd'): + self._platform = 'openbsd' + elif self._platform.startswith('solaris') or self._platform == 'sunos5': + self._platform = 'solaris' + elif self._platform.startswith('mingw'): + self._platform = 'mingw' + elif self._platform.startswith('win'): + self._platform = 'msvc' + elif self._platform.startswith('bitrig'): + self._platform = 'bitrig' + elif self._platform.startswith('netbsd'): + self._platform = 'netbsd' + + @staticmethod + def known_platforms(): + return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5', + 'mingw', 'msvc', 'gnukfreebsd', 'bitrig', 'netbsd'] + + def platform(self): + return self._platform + + def is_linux(self): + return self._platform == 'linux' + + def is_mingw(self): + return self._platform == 'mingw' + + def is_msvc(self): + return self._platform == 'msvc' + + def msvc_needs_fs(self): + popen = subprocess.Popen(['cl', '/nologo', '/?'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = popen.communicate() + return b'/FS ' in out + + def is_windows(self): + return self.is_mingw() or self.is_msvc() + + def is_solaris(self): + return self._platform == 'solaris' + + def uses_usr_local(self): + return self._platform in ('freebsd', 'openbsd', 'bitrig') + + def supports_ppoll(self): + return self._platform in ('linux', 'openbsd', 'bitrig') + + def supports_ninja_browse(self): + return not self.is_windows() and not self.is_solaris() + + +class Bootstrap: + """API shim for ninja_syntax.Writer that instead runs the commands. + + Used to bootstrap Ninja from scratch. In --bootstrap mode this + class is used to execute all the commands to build an executable. + It also proxies all calls to an underlying ninja_syntax.Writer, to + behave like non-bootstrap mode. + """ + def __init__(self, writer, verbose=False): + self.writer = writer + self.verbose = verbose + # Map of variable name => expanded variable value. + self.vars = {} + # Map of rule name => dict of rule attributes. + self.rules = { + 'phony': {} + } + + def comment(self, text): + return self.writer.comment(text) + + def newline(self): + return self.writer.newline() + + def variable(self, key, val): + self.vars[key] = self._expand(val) + return self.writer.variable(key, val) + + def rule(self, name, **kwargs): + self.rules[name] = kwargs + return self.writer.rule(name, **kwargs) + + def build(self, outputs, rule, inputs=None, **kwargs): + ruleattr = self.rules[rule] + cmd = ruleattr.get('command') + if cmd is None: # A phony rule, for example. + return + + # Implement just enough of Ninja variable expansion etc. to + # make the bootstrap build work. + local_vars = { + 'in': self._expand_paths(inputs), + 'out': self._expand_paths(outputs) + } + for key, val in kwargs.get('variables', []): + local_vars[key] = ' '.join(ninja_syntax.as_list(val)) + + self._run_command(self._expand(cmd, local_vars)) + + return self.writer.build(outputs, rule, inputs, **kwargs) + + def default(self, paths): + return self.writer.default(paths) + + def _expand_paths(self, paths): + """Expand $vars in an array of paths, e.g. from a 'build' block.""" + paths = ninja_syntax.as_list(paths) + return ' '.join(map(self._shell_escape, (map(self._expand, paths)))) + + def _expand(self, str, local_vars={}): + """Expand $vars in a string.""" + return ninja_syntax.expand(str, self.vars, local_vars) + + def _shell_escape(self, path): + """Quote paths containing spaces.""" + return '"%s"' % path if ' ' in path else path + + def _run_command(self, cmdline): + """Run a subcommand, quietly. Prints the full command on error.""" + try: + if self.verbose: + print(cmdline) + subprocess.check_call(cmdline, shell=True) + except subprocess.CalledProcessError: + print('when running: ', cmdline) + raise + + parser = OptionParser() profilers = ['gmon', 'pprof'] +parser.add_option('--bootstrap', action='store_true', + help='bootstrap a ninja binary from nothing') +parser.add_option('--verbose', action='store_true', + help='enable verbose build') parser.add_option('--platform', help='target platform (' + - '/'.join(platform_helper.platforms()) + ')', - choices=platform_helper.platforms()) + '/'.join(Platform.known_platforms()) + ')', + choices=Platform.known_platforms()) parser.add_option('--host', help='host platform (' + - '/'.join(platform_helper.platforms()) + ')', - choices=platform_helper.platforms()) + '/'.join(Platform.known_platforms()) + ')', + choices=Platform.known_platforms()) parser.add_option('--debug', action='store_true', help='enable debugging extras',) parser.add_option('--profile', metavar='TYPE', choices=profilers, help='enable profiling (' + '/'.join(profilers) + ')',) -parser.add_option('--with-gtest', metavar='PATH', - help='use gtest unpacked in directory PATH') +parser.add_option('--with-gtest', metavar='PATH', help='ignored') parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) @@ -57,15 +208,27 @@ if args: print('ERROR: extra unparsed command-line arguments:', args) sys.exit(1) -platform = platform_helper.Platform(options.platform) +platform = Platform(options.platform) if options.host: - host = platform_helper.Platform(options.host) + host = Platform(options.host) else: host = platform BUILD_FILENAME = 'build.ninja' -buildfile = open(BUILD_FILENAME, 'w') -n = ninja_syntax.Writer(buildfile) +ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w')) +n = ninja_writer + +if options.bootstrap: + # Make the build directory. + try: + os.mkdir('build') + except OSError: + pass + # Wrap ninja_writer with the Bootstrapper, which also executes the + # commands. + print('bootstrapping ninja...') + n = Bootstrap(n, verbose=options.verbose) + n.comment('This file is used to build ninja itself.') n.comment('It is generated by ' + os.path.basename(__file__) + '.') n.newline() @@ -74,11 +237,15 @@ n.variable('ninja_required_version', '1.3') n.newline() n.comment('The arguments passed to configure.py, for rerunning it.') -n.variable('configure_args', ' '.join(sys.argv[1:])) +configure_args = sys.argv[1:] +if '--bootstrap' in configure_args: + configure_args.remove('--bootstrap') +n.variable('configure_args', ' '.join(configure_args)) env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS']) configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys) if configure_env: - config_str = ' '.join([k + '=' + configure_env[k] for k in configure_env]) + config_str = ' '.join([k + '=' + pipes.quote(configure_env[k]) + for k in configure_env]) n.variable('configure_env', config_str + '$ ') n.newline() @@ -89,11 +256,11 @@ if platform.is_msvc(): objext = '.obj' def src(filename): - return os.path.join('src', filename) + return os.path.join('$sourcedir', 'src', filename) def built(filename): return os.path.join('$builddir', filename) def doc(filename): - return os.path.join('doc', filename) + return os.path.join('$sourcedir', 'doc', filename) def cc(name, **kwargs): return n.build(built(name + objext), 'cxx', src(name + '.c'), **kwargs) def cxx(name, **kwargs): @@ -105,6 +272,7 @@ def binary(name): return exe return name +n.variable('sourcedir', sourcedir) n.variable('builddir', 'build') n.variable('cxx', CXX) if platform.is_msvc(): @@ -113,7 +281,8 @@ else: n.variable('ar', configure_env.get('AR', 'ar')) if platform.is_msvc(): - cflags = ['/nologo', # Don't print startup banner. + cflags = ['/showIncludes', + '/nologo', # Don't print startup banner. '/Zi', # Create pdb with debug info. '/W4', # Highest warning level. '/WX', # Warnings as errors. @@ -121,13 +290,19 @@ if platform.is_msvc(): '/wd4512', '/wd4800', '/wd4702', '/wd4819', # Disable warnings about passing "this" during initialization. '/wd4355', + # Disable warnings about ignored typedef in DbgHelp.h + '/wd4091', '/GR-', # Disable RTTI. # Disable size_t -> int truncation warning. # We never have strings or arrays larger than 2**31. '/wd4267', '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', - '/D_VARIADIC_MAX=10', + '/D_HAS_EXCEPTIONS=0', '/DNINJA_PYTHON="%s"' % options.with_python] + if options.bootstrap: + # In bootstrap mode, we have no ninja process to catch /showIncludes + # output. + cflags.remove('/showIncludes') if platform.msvc_needs_fs(): cflags.append('/FS') ldflags = ['/DEBUG', '/libpath:$builddir'] @@ -148,17 +323,28 @@ else: cflags.remove('-fno-rtti') # Needed for above pedanticness. else: cflags += ['-O2', '-DNDEBUG'] - if 'clang' in os.path.basename(CXX): - cflags += ['-fcolor-diagnostics'] + try: + proc = subprocess.Popen( + [CXX, '-fdiagnostics-color', '-c', '-x', 'c++', '/dev/null', + '-o', '/dev/null'], + stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT) + if proc.wait() == 0: + cflags += ['-fdiagnostics-color'] + except: + pass if platform.is_mingw(): cflags += ['-D_WIN32_WINNT=0x0501'] ldflags = ['-L$builddir'] + if platform.uses_usr_local(): + cflags.append('-I/usr/local/include') + ldflags.append('-L/usr/local/lib') + libs = [] if platform.is_mingw(): cflags.remove('-fvisibility=hidden'); ldflags.append('-static') -elif platform.is_sunos5(): +elif platform.is_solaris(): cflags.remove('-fvisibility=hidden') elif platform.is_msvc(): pass @@ -170,9 +356,13 @@ else: cflags.append('-fno-omit-frame-pointer') libs.extend(['-Wl,--no-as-needed', '-lprofiler']) -if (platform.is_linux() or platform.is_openbsd() or platform.is_bitrig()) and \ - not options.force_pselect: +if platform.supports_ppoll() and not options.force_pselect: cflags.append('-DUSE_PPOLL') +if platform.supports_ninja_browse(): + cflags.append('-DNINJA_HAVE_BROWSE') + +# Search for generated headers relative to build dir. +cflags.append('-I.') def shell_escape(str): """Escape str such that it's interpreted as a single argument by @@ -195,9 +385,10 @@ n.newline() if platform.is_msvc(): n.rule('cxx', - command='$cxx /showIncludes $cflags -c $in /Fo$out', + command='$cxx $cflags -c $in /Fo$out', description='CXX $out', - deps='msvc') + deps='msvc' # /showIncludes is included in $cflags. + ) else: n.rule('cxx', command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out', @@ -232,13 +423,13 @@ n.newline() objs = [] -if not platform.is_windows() and not platform.is_solaris(): +if platform.supports_ninja_browse(): n.comment('browse_py.h is used to inline browse.py.') n.rule('inline', - command='src/inline.sh $varname < $in > $out', + command='"%s"' % src('inline.sh') + ' $varname < $in > $out', description='INLINE $out') n.build(built('browse_py.h'), 'inline', src('browse.py'), - implicit='src/inline.sh', + implicit=src('inline.sh'), variables=[('varname', 'kBrowsePy')]) n.newline() @@ -247,7 +438,6 @@ if not platform.is_windows() and not platform.is_solaris(): n.comment('the depfile parser and ninja lexers are generated using re2c.') def has_re2c(): - import subprocess try: proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE) return int(proc.communicate()[0], 10) >= 1103 @@ -316,37 +506,16 @@ ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, n.newline() all_targets += ninja +if options.bootstrap: + # We've built the ninja binary. Don't run any more commands + # through the bootstrap executor, but continue writing the + # build.ninja file. + n = ninja_writer + n.comment('Tests all build into ninja_test executable.') -variables = [] -test_cflags = cflags + ['-DGTEST_HAS_RTTI=0'] -test_ldflags = None -test_libs = libs objs = [] -if options.with_gtest: - path = options.with_gtest - - gtest_all_incs = '-I%s -I%s' % (path, os.path.join(path, 'include')) - if platform.is_msvc(): - gtest_cflags = '/nologo /EHsc /Zi /D_VARIADIC_MAX=10 ' - if platform.msvc_needs_fs(): - gtest_cflags += '/FS ' - gtest_cflags += gtest_all_incs - else: - gtest_cflags = '-fvisibility=hidden ' + gtest_all_incs - objs += n.build(built('gtest-all' + objext), 'cxx', - os.path.join(path, 'src', 'gtest-all.cc'), - variables=[('cflags', gtest_cflags)]) - - test_cflags.append('-I%s' % os.path.join(path, 'include')) -else: - # Use gtest from system. - if platform.is_msvc(): - test_libs.extend(['gtest_main.lib', 'gtest.lib']) - else: - test_libs.extend(['-lgtest_main', '-lgtest']) -n.variable('test_cflags', test_cflags) for name in ['build_log_test', 'build_test', 'clean_test', @@ -362,16 +531,13 @@ for name in ['build_log_test', 'subprocess_test', 'test', 'util_test']: - objs += cxx(name, variables=[('cflags', '$test_cflags')]) + objs += cxx(name) if platform.is_windows(): for name in ['includes_normalize_test', 'msvc_helper_test']: - objs += cxx(name, variables=[('cflags', test_cflags)]) + objs += cxx(name) -if not platform.is_windows(): - test_libs.append('-lpthread') ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, - variables=[('ldflags', test_ldflags), - ('libs', test_libs)]) + variables=[('libs', libs)]) n.newline() all_targets += ninja_test @@ -436,11 +602,12 @@ n.newline() if not host.is_mingw(): n.comment('Regenerate build files if build script changes.') n.rule('configure', - command='${configure_env}%s configure.py $configure_args' % + command='${configure_env}%s $sourcedir/configure.py $configure_args' % options.with_python, generator=True) n.build('build.ninja', 'configure', - implicit=['configure.py', os.path.normpath('misc/ninja_syntax.py')]) + implicit=['$sourcedir/configure.py', + os.path.normpath('$sourcedir/misc/ninja_syntax.py')]) n.newline() n.default(ninja) @@ -456,4 +623,20 @@ if host.is_linux(): n.build('all', 'phony', all_targets) +n.close() print('wrote %s.' % BUILD_FILENAME) + +verbose = '' +if options.verbose: + verbose = ' -v' + +if options.bootstrap: + print('bootstrap complete. rebuilding...') + if platform.is_windows(): + bootstrap_exe = 'ninja.bootstrap.exe' + if os.path.exists(bootstrap_exe): + os.unlink(bootstrap_exe) + os.rename('ninja.exe', bootstrap_exe) + subprocess.check_call('ninja.bootstrap.exe%s' % verbose, shell=True) + else: + subprocess.check_call('./ninja%s' % verbose, shell=True) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index fcf3db3..3c193f1 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -1,6 +1,5 @@ Ninja ===== -Evan Martin <martine@danga.com> Introduction @@ -180,6 +179,12 @@ Run `ninja`. By default, it looks for a file named `build.ninja` in the current directory and builds all out-of-date targets. You can specify which targets (files) to build as command line arguments. +There is also a special syntax `target^` for specifying a target +as the first output of some rule containing the source you put in +the command line, if one exists. For example, if you specify target as +`foo.c^` then `foo.o` will get built (assuming you have those targets +in your build files). + `ninja -h` prints help output. Many of Ninja's flags intentionally match those of Make; e.g `ninja -C build -j 20` changes into the `build` directory and runs 20 build commands in parallel. (Note that @@ -592,6 +597,11 @@ rule cc command = cl /showIncludes -c $in /Fo$out ---- +If the include directory directives are using absolute paths, your depfile +may result in a mixture of relative and absolute paths. Paths used by other +build rules need to match exactly. Therefore, it is recommended to use +relative paths in these cases. + [[ref_pool]] Pools ~~~~~ @@ -696,9 +706,7 @@ Lexical syntax Ninja is mostly encoding agnostic, as long as the bytes Ninja cares about (like slashes in paths) are ASCII. This means e.g. UTF-8 or -ISO-8859-1 input files ought to work. (To simplify some code, tabs -and carriage returns are currently disallowed; this could be fixed if -it really mattered to you.) +ISO-8859-1 input files ought to work. Comments begin with `#` and extend to the end of the line. @@ -885,6 +893,9 @@ header file before starting a subsequent compilation step. (Once the header is used in compilation, a generated dependency file will then express the implicit dependency.) +File paths are compared as is, which means that an absolute path and a +relative path, pointing to the same file, are considered different by Ninja. + Variable expansion ~~~~~~~~~~~~~~~~~~ @@ -916,9 +927,12 @@ Evaluation and scoping Top-level variable declarations are scoped to the file they occur in. +Rule declarations are also scoped to the file they occur in. +_(Available since Ninja 1.6)_ + The `subninja` keyword, used to include another `.ninja` file, introduces a new scope. The included `subninja` file may use the -variables from the parent file, and shadow their values for the file's +variables and rules from the parent file, and shadow their values for the file's scope, but it won't affect values of the variables in the parent. To include another `.ninja` file in the current scope, much like a C diff --git a/misc/afl-fuzz-tokens/kw_build b/misc/afl-fuzz-tokens/kw_build new file mode 100644 index 0000000..c795b05 --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_build @@ -0,0 +1 @@ +build
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/kw_default b/misc/afl-fuzz-tokens/kw_default new file mode 100644 index 0000000..331d858 --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_default @@ -0,0 +1 @@ +default
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/kw_include b/misc/afl-fuzz-tokens/kw_include new file mode 100644 index 0000000..2996fba --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_include @@ -0,0 +1 @@ +include
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/kw_pool b/misc/afl-fuzz-tokens/kw_pool new file mode 100644 index 0000000..e783591 --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_pool @@ -0,0 +1 @@ +pool
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/kw_rule b/misc/afl-fuzz-tokens/kw_rule new file mode 100644 index 0000000..841e840 --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_rule @@ -0,0 +1 @@ +rule
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/kw_subninja b/misc/afl-fuzz-tokens/kw_subninja new file mode 100644 index 0000000..c4fe0c7 --- /dev/null +++ b/misc/afl-fuzz-tokens/kw_subninja @@ -0,0 +1 @@ +subninja
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_a b/misc/afl-fuzz-tokens/misc_a new file mode 100644 index 0000000..2e65efe --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_a @@ -0,0 +1 @@ +a
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_b b/misc/afl-fuzz-tokens/misc_b new file mode 100644 index 0000000..63d8dbd --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_b @@ -0,0 +1 @@ +b
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_colon b/misc/afl-fuzz-tokens/misc_colon new file mode 100644 index 0000000..22ded55 --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_colon @@ -0,0 +1 @@ +:
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_cont b/misc/afl-fuzz-tokens/misc_cont new file mode 100644 index 0000000..857f13a --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_cont @@ -0,0 +1 @@ +$ diff --git a/misc/afl-fuzz-tokens/misc_dollar b/misc/afl-fuzz-tokens/misc_dollar new file mode 100644 index 0000000..6f4f765 --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_dollar @@ -0,0 +1 @@ +$
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_eq b/misc/afl-fuzz-tokens/misc_eq new file mode 100644 index 0000000..851c75c --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_eq @@ -0,0 +1 @@ +=
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_indent b/misc/afl-fuzz-tokens/misc_indent new file mode 100644 index 0000000..136d063 --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_indent @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_pipe b/misc/afl-fuzz-tokens/misc_pipe new file mode 100644 index 0000000..a3871d4 --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_pipe @@ -0,0 +1 @@ +|
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_pipepipe b/misc/afl-fuzz-tokens/misc_pipepipe new file mode 100644 index 0000000..27cc728 --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_pipepipe @@ -0,0 +1 @@ +||
\ No newline at end of file diff --git a/misc/afl-fuzz-tokens/misc_space b/misc/afl-fuzz-tokens/misc_space new file mode 100644 index 0000000..0519ecb --- /dev/null +++ b/misc/afl-fuzz-tokens/misc_space @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/misc/afl-fuzz/build.ninja b/misc/afl-fuzz/build.ninja new file mode 100644 index 0000000..52cd2f1 --- /dev/null +++ b/misc/afl-fuzz/build.ninja @@ -0,0 +1,5 @@ +rule b + command = clang -MMD -MF $out.d -o $out -c $in + description = building $out + +build a.o: b a.c diff --git a/misc/bash-completion b/misc/bash-completion index 6edf4df..0536760 100644 --- a/misc/bash-completion +++ b/misc/bash-completion @@ -50,7 +50,7 @@ _ninja_target() { esac done; targets_command="eval ninja -C \"${dir}\" -t targets all" - targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}') + targets=$(${targets_command} 2>/dev/null | awk -F: '{print $1}') COMPREPLY=($(compgen -W "$targets" -- "$cur")) fi return diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el index 9fe6fc8..639e537 100644 --- a/misc/ninja-mode.el +++ b/misc/ninja-mode.el @@ -1,4 +1,6 @@ -;;; ninja-mode.el --- Major mode for editing .ninja files +;;; ninja-mode.el --- Major mode for editing .ninja files -*- lexical-binding: t -*- + +;; Package-Requires: ((emacs "24")) ;; Copyright 2011 Google Inc. All Rights Reserved. ;; @@ -22,26 +24,57 @@ ;;; Code: (defvar ninja-keywords - (list - '("^#.*" . font-lock-comment-face) - (cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include" - "pool" "default") - 'words)) - font-lock-keyword-face) - '("\\([[:alnum:]_]+\\) =" . (1 font-lock-variable-name-face)) - ;; Variable expansion. - '("\\($[[:alnum:]_]+\\)" . (1 font-lock-variable-name-face)) - ;; Rule names - '("rule \\([[:alnum:]_]+\\)" . (1 font-lock-function-name-face)) - )) - -;;;###autoload -(define-derived-mode ninja-mode fundamental-mode "ninja" - (setq comment-start "#") - ; Pass extra "t" to turn off syntax-based fontification -- we don't want - ; quoted strings highlighted. - (setq font-lock-defaults '(ninja-keywords t)) - ) + `((,(concat "^" (regexp-opt '("rule" "build" "subninja" "include" + "pool" "default") + 'words)) + . font-lock-keyword-face) + ("\\([[:alnum:]_]+\\) =" 1 font-lock-variable-name-face) + ;; Variable expansion. + ("$[[:alnum:]_]+" . font-lock-variable-name-face) + ("${[[:alnum:]._]+}" . font-lock-variable-name-face) + ;; Rule names + ("rule +\\([[:alnum:]_.-]+\\)" 1 font-lock-function-name-face) + ;; Build Statement - highlight the rule used, + ;; allow for escaped $,: in outputs. + ("build +\\(?:[^:$\n]\\|$[:$]\\)+ *: *\\([[:alnum:]_.-]+\\)" + 1 font-lock-function-name-face))) + +(defvar ninja-mode-syntax-table + (let ((table (make-syntax-table))) + (modify-syntax-entry ?\" "." table) + table) + "Syntax table used in `ninja-mode'.") + +(defun ninja-syntax-propertize (start end) + (save-match-data + (goto-char start) + (while (search-forward "#" end t) + (let ((match-pos (match-beginning 0))) + (when (and + ;; Is it the first non-white character on the line? + (eq match-pos (save-excursion (back-to-indentation) (point))) + (save-excursion + (goto-char (line-end-position 0)) + (or + ;; If we're continuting the previous line, it's not a + ;; comment. + (not (eq ?$ (char-before))) + ;; Except if the previous line is a comment as well, as the + ;; continuation dollar is ignored then. + (nth 4 (syntax-ppss))))) + (put-text-property match-pos (1+ match-pos) 'syntax-table '(11)) + (let ((line-end (line-end-position))) + ;; Avoid putting properties past the end of the buffer. + ;; Otherwise we get an `args-out-of-range' error. + (unless (= line-end (1+ (buffer-size))) + (put-text-property line-end (1+ line-end) 'syntax-table '(12))))))))) + +;;;###autoload +(define-derived-mode ninja-mode prog-mode "ninja" + (set (make-local-variable 'comment-start) "#") + (set (make-local-variable 'parse-sexp-lookup-properties) t) + (set (make-local-variable 'syntax-propertize-function) #'ninja-syntax-propertize) + (setq font-lock-defaults '(ninja-keywords))) ;; Run ninja-mode for files ending in .ninja. ;;;###autoload diff --git a/misc/ninja.vim b/misc/ninja.vim index 9e6ee5c..190d9ce 100644 --- a/misc/ninja.vim +++ b/misc/ninja.vim @@ -1,16 +1,16 @@ " ninja build file syntax. " Language: ninja build file as described at -" http://martine.github.com/ninja/manual.html +" http://ninja-build.org/manual.html " Version: 1.4 " Last Change: 2014/05/13 " Maintainer: Nicolas Weber <nicolasweber@gmx.de> -" Version 1.3 of this script is in the upstream vim repository and will be +" Version 1.4 of this script is in the upstream vim repository and will be " included in the next vim release. If you change this, please send your change " upstream. " ninja lexer and parser are at -" https://github.com/martine/ninja/blob/master/src/lexer.in.cc -" https://github.com/martine/ninja/blob/master/src/manifest_parser.cc +" https://github.com/ninja-build/ninja/blob/master/src/lexer.in.cc +" https://github.com/ninja-build/ninja/blob/master/src/manifest_parser.cc if exists("b:current_syntax") finish diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index 14b932f..8673518 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -7,6 +7,7 @@ just a helpful utility for build-file-generation systems that already use Python. """ +import re import textwrap def escape_path(word): @@ -59,16 +60,16 @@ class Writer(object): def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, variables=None): - outputs = self._as_list(outputs) + outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] - all_inputs = [escape_path(x) for x in self._as_list(inputs)] + all_inputs = [escape_path(x) for x in as_list(inputs)] if implicit: - implicit = [escape_path(x) for x in self._as_list(implicit)] + implicit = [escape_path(x) for x in as_list(implicit)] all_inputs.append('|') all_inputs.extend(implicit) if order_only: - order_only = [escape_path(x) for x in self._as_list(order_only)] + order_only = [escape_path(x) for x in as_list(order_only)] all_inputs.append('||') all_inputs.extend(order_only) @@ -93,7 +94,7 @@ class Writer(object): self._line('subninja %s' % path) def default(self, paths): - self._line('default %s' % ' '.join(self._as_list(paths))) + self._line('default %s' % ' '.join(as_list(paths))) def _count_dollars_before_index(self, s, i): """Returns the number of '$' characters right in front of s[i].""" @@ -140,12 +141,16 @@ class Writer(object): self.output.write(leading_space + text + '\n') - def _as_list(self, input): - if input is None: - return [] - if isinstance(input, list): - return input - return [input] + def close(self): + self.output.close() + + +def as_list(input): + if input is None: + return [] + if isinstance(input, list): + return input + return [input] def escape(string): @@ -154,3 +159,17 @@ def escape(string): assert '\n' not in string, 'Ninja syntax does not allow newlines' # We only have one special metacharacter: '$'. return string.replace('$', '$$') + + +def expand(string, vars, local_vars={}): + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m): + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string) diff --git a/misc/ninja_syntax_test.py b/misc/ninja_syntax_test.py index 2aef7ff..36b2e7b 100755 --- a/misc/ninja_syntax_test.py +++ b/misc/ninja_syntax_test.py @@ -148,5 +148,31 @@ build out: cc in ''', self.out.getvalue()) +class TestExpand(unittest.TestCase): + def test_basic(self): + vars = {'x': 'X'} + self.assertEqual('foo', ninja_syntax.expand('foo', vars)) + + def test_var(self): + vars = {'xyz': 'XYZ'} + self.assertEqual('fooXYZ', ninja_syntax.expand('foo$xyz', vars)) + + def test_vars(self): + vars = {'x': 'X', 'y': 'YYY'} + self.assertEqual('XYYY', ninja_syntax.expand('$x$y', vars)) + + def test_space(self): + vars = {} + self.assertEqual('x y z', ninja_syntax.expand('x$ y$ z', vars)) + + def test_locals(self): + vars = {'x': 'a'} + local_vars = {'x': 'b'} + self.assertEqual('a', ninja_syntax.expand('$x', vars)) + self.assertEqual('b', ninja_syntax.expand('$x', vars, local_vars)) + + def test_double(self): + self.assertEqual('a b$c', ninja_syntax.expand('a$ b$$c', {})) + if __name__ == '__main__': unittest.main() diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec index f0c46fe..05f5a07 100644 --- a/misc/packaging/ninja.spec +++ b/misc/packaging/ninja.spec @@ -4,7 +4,7 @@ Version: %{ver} Release: %{rel}%{?dist} Group: Development/Tools License: Apache 2.0 -URL: https://github.com/martine/ninja +URL: https://github.com/ninja-build/ninja Source0: %{name}-%{version}-%{rel}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{rel} @@ -23,7 +23,7 @@ seconds to start building after changing one file. Ninja is under a second. %build echo Building.. -./bootstrap.py +./configure.py --bootstrap ./ninja manual %install diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py index 837007e..ca49535 100644 --- a/misc/write_fake_manifests.py +++ b/misc/write_fake_manifests.py @@ -182,7 +182,7 @@ def FileWriter(path): def random_targets(): - num_targets = 800 + num_targets = 1500 gen = GenRandom() # N-1 static libraries, and 1 executable depending on all of them. diff --git a/misc/zsh-completion b/misc/zsh-completion index 2fe16fb..fd9b3a7 100644 --- a/misc/zsh-completion +++ b/misc/zsh-completion @@ -17,7 +17,14 @@ # . path/to/ninja/misc/zsh-completion __get_targets() { - ninja -t targets 2>/dev/null | while read -r a b; do echo $a | cut -d ':' -f1; done; + dir="." + if [ -n "${opt_args[-C]}" ]; + then + eval dir="${opt_args[-C]}" + fi + targets_command="ninja -C \"${dir}\" -t targets" + eval ${targets_command} 2>/dev/null | while read -r a b; do echo $a | cut -d ':' -f1; done; + } __get_tools() { @@ -25,7 +32,7 @@ __get_tools() { } __get_modes() { - ninja -d list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2 | head -n -1 + ninja -d list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2 | sed '$d' } __modes() { diff --git a/platform_helper.py b/platform_helper.py deleted file mode 100644 index bc3a125..0000000 --- a/platform_helper.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# Copyright 2011 Google Inc. -# Copyright 2013 Patrick von Reth <vonreth@kde.org> -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys - -def platforms(): - return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5', - 'mingw', 'msvc', 'gnukfreebsd', 'bitrig'] - -class Platform(object): - def __init__(self, platform): - self._platform = platform - if not self._platform is None: - return - self._platform = sys.platform - if self._platform.startswith('linux'): - self._platform = 'linux' - elif self._platform.startswith('freebsd'): - self._platform = 'freebsd' - elif self._platform.startswith('gnukfreebsd'): - self._platform = 'freebsd' - elif self._platform.startswith('openbsd'): - self._platform = 'openbsd' - elif self._platform.startswith('solaris'): - self._platform = 'solaris' - elif self._platform.startswith('mingw'): - self._platform = 'mingw' - elif self._platform.startswith('win'): - self._platform = 'msvc' - elif self._platform.startswith('bitrig'): - self._platform = 'bitrig' - - def platform(self): - return self._platform - - def is_linux(self): - return self._platform == 'linux' - - def is_mingw(self): - return self._platform == 'mingw' - - def is_msvc(self): - return self._platform == 'msvc' - - def msvc_needs_fs(self): - import subprocess - popen = subprocess.Popen(['cl', '/nologo', '/?'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = popen.communicate() - return '/FS ' in str(out) - - def is_windows(self): - return self.is_mingw() or self.is_msvc() - - def is_solaris(self): - return self._platform == 'solaris' - - def is_freebsd(self): - return self._platform == 'freebsd' - - def is_openbsd(self): - return self._platform == 'openbsd' - - def is_sunos5(self): - return self._platform == 'sunos5' - - def is_bitrig(self): - return self._platform == 'bitrig' diff --git a/src/browse.cc b/src/browse.cc index 83bfe43..8673919 100644 --- a/src/browse.cc +++ b/src/browse.cc @@ -18,7 +18,7 @@ #include <stdlib.h> #include <unistd.h> -#include "../build/browse_py.h" +#include "build/browse_py.h" void RunBrowsePython(State* state, const char* ninja_command, const char* initial_target) { diff --git a/src/build.cc b/src/build.cc index 64bcea3..b8c2560 100644 --- a/src/build.cc +++ b/src/build.cc @@ -278,7 +278,7 @@ bool Plan::AddSubTarget(Node* node, vector<Node*>* stack, string* err) { return false; } - if (CheckDependencyCycle(node, stack, err)) + if (CheckDependencyCycle(node, *stack, err)) return false; if (edge->outputs_ready()) @@ -316,20 +316,32 @@ bool Plan::AddSubTarget(Node* node, vector<Node*>* stack, string* err) { return true; } -bool Plan::CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err) { - vector<Node*>::reverse_iterator ri = - find(stack->rbegin(), stack->rend(), node); - if (ri == stack->rend()) +bool Plan::CheckDependencyCycle(Node* node, const vector<Node*>& stack, + string* err) { + vector<Node*>::const_iterator start = stack.begin(); + while (start != stack.end() && (*start)->in_edge() != node->in_edge()) + ++start; + if (start == stack.end()) return false; - // Add this node onto the stack to make it clearer where the loop - // is. - stack->push_back(node); + // Build error string for the cycle. + vector<Node*> cycle(start, stack.end()); + cycle.push_back(node); + + if (cycle.front() != cycle.back()) { + // Consider + // build a b: cat c + // build c: cat a + // stack will contain [b, c], node will be a. To not print b -> c -> a, + // shift by one to get c -> a -> c which makes the cycle clear. + cycle.erase(cycle.begin()); + cycle.push_back(cycle.front()); + assert(cycle.front() == cycle.back()); + } - vector<Node*>::iterator start = find(stack->begin(), stack->end(), node); *err = "dependency cycle: "; - for (vector<Node*>::iterator i = start; i != stack->end(); ++i) { - if (i != start) + for (vector<Node*>::const_iterator i = cycle.begin(); i != cycle.end(); ++i) { + if (i != cycle.begin()) err->append(" -> "); err->append((*i)->path()); } @@ -339,9 +351,9 @@ bool Plan::CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err) { Edge* Plan::FindWork() { if (ready_.empty()) return NULL; - set<Edge*>::iterator i = ready_.begin(); - Edge* edge = *i; - ready_.erase(i); + set<Edge*>::iterator e = ready_.begin(); + Edge* edge = *e; + ready_.erase(e); return edge; } @@ -350,7 +362,7 @@ void Plan::ScheduleWork(Edge* edge) { if (pool->ShouldDelayEdge()) { // The graph is not completely clean. Some Nodes have duplicate Out edges. // We need to explicitly ignore these here, otherwise their work will get - // scheduled twice (see https://github.com/martine/ninja/pull/519) + // scheduled twice (see https://github.com/ninja-build/ninja/pull/519) if (ready_.count(edge)) { return; } @@ -362,101 +374,106 @@ void Plan::ScheduleWork(Edge* edge) { } } -void Plan::ResumeDelayedJobs(Edge* edge) { - edge->pool()->EdgeFinished(*edge); - edge->pool()->RetrieveReadyEdges(&ready_); -} - void Plan::EdgeFinished(Edge* edge) { - map<Edge*, bool>::iterator i = want_.find(edge); - assert(i != want_.end()); - if (i->second) + map<Edge*, bool>::iterator e = want_.find(edge); + assert(e != want_.end()); + bool directly_wanted = e->second; + if (directly_wanted) --wanted_edges_; - want_.erase(i); + want_.erase(e); edge->outputs_ready_ = true; - // See if this job frees up any delayed jobs - ResumeDelayedJobs(edge); + // See if this job frees up any delayed jobs. + if (directly_wanted) + edge->pool()->EdgeFinished(*edge); + edge->pool()->RetrieveReadyEdges(&ready_); // Check off any nodes we were waiting for with this edge. - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - NodeFinished(*i); + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + NodeFinished(*o); } } void Plan::NodeFinished(Node* node) { // See if we we want any edges from this node. - for (vector<Edge*>::const_iterator i = node->out_edges().begin(); - i != node->out_edges().end(); ++i) { - map<Edge*, bool>::iterator want_i = want_.find(*i); - if (want_i == want_.end()) + for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); + oe != node->out_edges().end(); ++oe) { + map<Edge*, bool>::iterator want_e = want_.find(*oe); + if (want_e == want_.end()) continue; // See if the edge is now ready. - if ((*i)->AllInputsReady()) { - if (want_i->second) { - ScheduleWork(*i); + if ((*oe)->AllInputsReady()) { + if (want_e->second) { + ScheduleWork(*oe); } else { // We do not need to build this edge, but we might need to build one of // its dependents. - EdgeFinished(*i); + EdgeFinished(*oe); } } } } -void Plan::CleanNode(DependencyScan* scan, Node* node) { +bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { node->set_dirty(false); - for (vector<Edge*>::const_iterator ei = node->out_edges().begin(); - ei != node->out_edges().end(); ++ei) { + for (vector<Edge*>::const_iterator oe = node->out_edges().begin(); + oe != node->out_edges().end(); ++oe) { // Don't process edges that we don't actually want. - map<Edge*, bool>::iterator want_i = want_.find(*ei); - if (want_i == want_.end() || !want_i->second) + map<Edge*, bool>::iterator want_e = want_.find(*oe); + if (want_e == want_.end() || !want_e->second) continue; // Don't attempt to clean an edge if it failed to load deps. - if ((*ei)->deps_missing_) + if ((*oe)->deps_missing_) continue; // If all non-order-only inputs for this edge are now clean, // we might have changed the dirty state of the outputs. vector<Node*>::iterator - begin = (*ei)->inputs_.begin(), - end = (*ei)->inputs_.end() - (*ei)->order_only_deps_; + begin = (*oe)->inputs_.begin(), + end = (*oe)->inputs_.end() - (*oe)->order_only_deps_; if (find_if(begin, end, mem_fun(&Node::dirty)) == end) { // Recompute most_recent_input. Node* most_recent_input = NULL; - for (vector<Node*>::iterator ni = begin; ni != end; ++ni) { - if (!most_recent_input || (*ni)->mtime() > most_recent_input->mtime()) - most_recent_input = *ni; + for (vector<Node*>::iterator i = begin; i != end; ++i) { + if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) + most_recent_input = *i; } // Now, this edge is dirty if any of the outputs are dirty. // If the edge isn't dirty, clean the outputs and mark the edge as not // wanted. - if (!scan->RecomputeOutputsDirty(*ei, most_recent_input)) { - for (vector<Node*>::iterator ni = (*ei)->outputs_.begin(); - ni != (*ei)->outputs_.end(); ++ni) { - CleanNode(scan, *ni); + bool outputs_dirty = false; + if (!scan->RecomputeOutputsDirty(*oe, most_recent_input, + &outputs_dirty, err)) { + return false; + } + if (!outputs_dirty) { + for (vector<Node*>::iterator o = (*oe)->outputs_.begin(); + o != (*oe)->outputs_.end(); ++o) { + if (!CleanNode(scan, *o, err)) + return false; } - want_i->second = false; + want_e->second = false; --wanted_edges_; - if (!(*ei)->is_phony()) + if (!(*oe)->is_phony()) --command_edges_; } } } + return true; } void Plan::Dump() { printf("pending: %d\n", (int)want_.size()); - for (map<Edge*, bool>::iterator i = want_.begin(); i != want_.end(); ++i) { - if (i->second) + for (map<Edge*, bool>::iterator e = want_.begin(); e != want_.end(); ++e) { + if (e->second) printf("want "); - i->first->Dump(); + e->first->Dump(); } printf("ready: %d\n", (int)ready_.size()); } @@ -477,9 +494,9 @@ struct RealCommandRunner : public CommandRunner { vector<Edge*> RealCommandRunner::GetActiveEdges() { vector<Edge*> edges; - for (map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.begin(); - i != subproc_to_edge_.end(); ++i) - edges.push_back(i->second); + for (map<Subprocess*, Edge*>::iterator e = subproc_to_edge_.begin(); + e != subproc_to_edge_.end(); ++e) + edges.push_back(e->second); return edges; } @@ -488,7 +505,9 @@ void RealCommandRunner::Abort() { } bool RealCommandRunner::CanRunMore() { - return ((int)subprocs_.running_.size()) < config_.parallelism + size_t subproc_number = + subprocs_.running_.size() + subprocs_.finished_.size(); + return (int)subproc_number < config_.parallelism && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) || GetLoadAverage() < config_.max_load_average); } @@ -514,9 +533,9 @@ bool RealCommandRunner::WaitForCommand(Result* result) { result->status = subproc->Finish(); result->output = subproc->GetOutput(); - map<Subprocess*, Edge*>::iterator i = subproc_to_edge_.find(subproc); - result->edge = i->second; - subproc_to_edge_.erase(i); + map<Subprocess*, Edge*>::iterator e = subproc_to_edge_.find(subproc); + result->edge = e->second; + subproc_to_edge_.erase(e); delete subproc; return true; @@ -539,11 +558,11 @@ void Builder::Cleanup() { vector<Edge*> active_edges = command_runner_->GetActiveEdges(); command_runner_->Abort(); - for (vector<Edge*>::iterator i = active_edges.begin(); - i != active_edges.end(); ++i) { - string depfile = (*i)->GetUnescapedDepfile(); - for (vector<Node*>::iterator ni = (*i)->outputs_.begin(); - ni != (*i)->outputs_.end(); ++ni) { + for (vector<Edge*>::iterator e = active_edges.begin(); + e != active_edges.end(); ++e) { + string depfile = (*e)->GetUnescapedDepfile(); + for (vector<Node*>::iterator o = (*e)->outputs_.begin(); + o != (*e)->outputs_.end(); ++o) { // Only delete this output if it was actually modified. This is // important for things like the generator where we don't want to // delete the manifest file if we can avoid it. But if the rule @@ -551,10 +570,12 @@ void Builder::Cleanup() { // need to rebuild an output because of a modified header file // mentioned in a depfile, and the command touches its depfile // but is interrupted before it touches its output file.) - if (!depfile.empty() || - (*ni)->mtime() != disk_interface_->Stat((*ni)->path())) { - disk_interface_->RemoveFile((*ni)->path()); - } + string err; + TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err); + if (new_mtime == -1) // Log and ignore Stat() errors. + Error("%s", err.c_str()); + if (!depfile.empty() || (*o)->mtime() != new_mtime) + disk_interface_->RemoveFile((*o)->path()); } if (!depfile.empty()) disk_interface_->RemoveFile(depfile); @@ -574,7 +595,6 @@ Node* Builder::AddTarget(const string& name, string* err) { } bool Builder::AddTarget(Node* node, string* err) { - node->StatIfNecessary(disk_interface_); if (Edge* in_edge = node->in_edge()) { if (!scan_.RecomputeDirty(in_edge, err)) return false; @@ -688,9 +708,9 @@ bool Builder::StartEdge(Edge* edge, string* err) { // Create directories necessary for outputs. // XXX: this will block; do we care? - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - if (!disk_interface_->MakeDirs((*i)->path())) + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + if (!disk_interface_->MakeDirs((*o)->path())) return false; } @@ -750,14 +770,17 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (edge->GetBindingBool("restat") && !config_.dry_run) { bool node_cleaned = false; - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - TimeStamp new_mtime = disk_interface_->Stat((*i)->path()); - if ((*i)->mtime() == new_mtime) { + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); + if (new_mtime == -1) + return false; + if ((*o)->mtime() == new_mtime) { // The rule command did not change the output. Propagate the clean // state through the build graph. // Note that this also applies to nonexistent outputs (mtime == 0). - plan_.CleanNode(&scan_, *i); + if (!plan_.CleanNode(&scan_, *o, err)) + return false; node_cleaned = true; } } @@ -767,14 +790,18 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { // (existing) non-order-only input or the depfile. for (vector<Node*>::iterator i = edge->inputs_.begin(); i != edge->inputs_.end() - edge->order_only_deps_; ++i) { - TimeStamp input_mtime = disk_interface_->Stat((*i)->path()); + TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err); + if (input_mtime == -1) + return false; if (input_mtime > restat_mtime) restat_mtime = input_mtime; } string depfile = edge->GetUnescapedDepfile(); if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { - TimeStamp depfile_mtime = disk_interface_->Stat(depfile); + TimeStamp depfile_mtime = disk_interface_->Stat(depfile, err); + if (depfile_mtime == -1) + return false; if (depfile_mtime > restat_mtime) restat_mtime = depfile_mtime; } @@ -803,7 +830,9 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { if (!deps_type.empty() && !config_.dry_run) { assert(edge->outputs_.size() == 1 && "should have been rejected by parser"); Node* out = edge->outputs_[0]; - TimeStamp deps_mtime = disk_interface_->Stat(out->path()); + TimeStamp deps_mtime = disk_interface_->Stat(out->path(), err); + if (deps_mtime == -1) + return false; if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) { *err = string("Error writing to deps log: ") + strerror(errno); return false; @@ -820,10 +849,17 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, #ifdef _WIN32 if (deps_type == "msvc") { CLParser parser; - result->output = parser.Parse(result->output, deps_prefix); + string output; + if (!parser.Parse(result->output, deps_prefix, &output, err)) + return false; + result->output = output; for (set<string>::iterator i = parser.includes_.begin(); i != parser.includes_.end(); ++i) { - deps_nodes->push_back(state_->GetNode(*i)); + // ~0 is assuming that with MSVC-parsed headers, it's ok to always make + // all backslashes (as some of the slashes will certainly be backslashes + // anyway). This could be fixed if necessary with some additional + // complexity in IncludesNormalize::Relativize. + deps_nodes->push_back(state_->GetNode(*i, ~0u)); } } else #endif @@ -848,9 +884,11 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, deps_nodes->reserve(deps.ins_.size()); for (vector<StringPiece>::iterator i = deps.ins_.begin(); i != deps.ins_.end(); ++i) { - if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) + unsigned int slash_bits; + if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits, + err)) return false; - deps_nodes->push_back(state_->GetNode(*i)); + deps_nodes->push_back(state_->GetNode(*i, slash_bits)); } if (disk_interface_->RemoveFile(depfile) < 0) { diff --git a/src/build.h b/src/build.h index eb3636a..8106faa 100644 --- a/src/build.h +++ b/src/build.h @@ -61,14 +61,16 @@ struct Plan { void EdgeFinished(Edge* edge); /// Clean the given node during the build. - void CleanNode(DependencyScan* scan, Node* node); + /// Return false on error. + bool CleanNode(DependencyScan* scan, Node* node, string* err); /// Number of edges with commands to run. int command_edge_count() const { return command_edges_; } private: bool AddSubTarget(Node* node, vector<Node*>* stack, string* err); - bool CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err); + bool CheckDependencyCycle(Node* node, const vector<Node*>& stack, + string* err); void NodeFinished(Node* node); /// Submits a ready edge as a candidate for execution. @@ -76,11 +78,6 @@ private: /// currently-full pool. void ScheduleWork(Edge* edge); - /// Allows jobs blocking on |edge| to potentially resume. - /// For example, if |edge| is a member of a pool, calling this may schedule - /// previously pending jobs in that pool. - void ResumeDelayedJobs(Edge* edge); - /// Keep track of which edges we want to build in this plan. If this map does /// not contain an entry for an edge, we do not want to build the entry or its /// dependents. If an entry maps to false, we do not want to build it, but we diff --git a/src/build_log.cc b/src/build_log.cc index 3f24c16..589c6da 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -19,7 +19,9 @@ #include <string.h> #ifndef _WIN32 +#ifndef __STDC_FORMAT_MACROS #define __STDC_FORMAT_MACROS +#endif #include <inttypes.h> #include <unistd.h> #endif @@ -354,7 +356,6 @@ bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { bool BuildLog::Recompact(const string& path, const BuildLogUser& user, string* err) { METRIC_RECORD(".ninja_log recompact"); - printf("Recompacting log...\n"); Close(); string temp_path = path + ".recompact"; diff --git a/src/build_log.h b/src/build_log.h index fe81a85..785961e 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -27,7 +27,7 @@ struct Edge; /// Can answer questions about the manifest for the BuildLog. struct BuildLogUser { - /// Return if a given output no longer part of the build manifest. + /// Return if a given output is no longer part of the build manifest. /// This is only called during recompaction and doesn't have to be fast. virtual bool IsPathDead(StringPiece s) const = 0; }; diff --git a/src/build_log_test.cc b/src/build_log_test.cc index 6738c7b..2c41ba6 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -17,12 +17,12 @@ #include "util.h" #include "test.h" +#include <sys/stat.h> #ifdef _WIN32 #include <fcntl.h> #include <share.h> #else #include <sys/types.h> -#include <sys/stat.h> #include <unistd.h> #endif diff --git a/src/build_test.cc b/src/build_test.cc index dad69dc..20fb664 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -14,6 +14,8 @@ #include "build.h" +#include <assert.h> + #include "build_log.h" #include "deps_log.h" #include "graph.h" @@ -49,9 +51,9 @@ struct PlanTest : public StateTestWithBuiltinRules { }; TEST_F(PlanTest, Basic) { - AssertParse(&state_, + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build out: cat mid\n" -"build mid: cat in\n"); +"build mid: cat in\n")); GetNode("mid")->MarkDirty(); GetNode("out")->MarkDirty(); string err; @@ -82,9 +84,9 @@ TEST_F(PlanTest, Basic) { // Test that two outputs from one rule can be handled as inputs to the next. TEST_F(PlanTest, DoubleOutputDirect) { - AssertParse(&state_, + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build out: cat mid1 mid2\n" -"build mid1 mid2: cat in\n"); +"build mid1 mid2: cat in\n")); GetNode("mid1")->MarkDirty(); GetNode("mid2")->MarkDirty(); GetNode("out")->MarkDirty(); @@ -109,11 +111,11 @@ TEST_F(PlanTest, DoubleOutputDirect) { // Test that two outputs from one rule can eventually be routed to another. TEST_F(PlanTest, DoubleOutputIndirect) { - AssertParse(&state_, + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build out: cat b1 b2\n" "build b1: cat a1\n" "build b2: cat a2\n" -"build a1 a2: cat in\n"); +"build a1 a2: cat in\n")); GetNode("a1")->MarkDirty(); GetNode("a2")->MarkDirty(); GetNode("b1")->MarkDirty(); @@ -147,11 +149,11 @@ TEST_F(PlanTest, DoubleOutputIndirect) { // Test that two edges from one output can both execute. TEST_F(PlanTest, DoubleDependent) { - AssertParse(&state_, + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build out: cat a1 a2\n" "build a1: cat mid\n" "build a2: cat mid\n" -"build mid: cat in\n"); +"build mid: cat in\n")); GetNode("mid")->MarkDirty(); GetNode("a1")->MarkDirty(); GetNode("a2")->MarkDirty(); @@ -184,11 +186,11 @@ TEST_F(PlanTest, DoubleDependent) { } TEST_F(PlanTest, DependencyCycle) { - AssertParse(&state_, + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build out: cat mid\n" "build mid: cat in\n" "build in: cat pre\n" -"build pre: cat out\n"); +"build pre: cat out\n")); GetNode("out")->MarkDirty(); GetNode("mid")->MarkDirty(); GetNode("in")->MarkDirty(); @@ -199,6 +201,43 @@ TEST_F(PlanTest, DependencyCycle) { ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } +TEST_F(PlanTest, CycleInEdgesButNotInNodes1) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build a b: cat a\n")); + EXPECT_FALSE(plan_.AddTarget(GetNode("b"), &err)); + ASSERT_EQ("dependency cycle: a -> a", err); +} + +TEST_F(PlanTest, CycleInEdgesButNotInNodes2) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build b a: cat a\n")); + EXPECT_FALSE(plan_.AddTarget(GetNode("b"), &err)); + ASSERT_EQ("dependency cycle: a -> a", err); +} + +TEST_F(PlanTest, CycleInEdgesButNotInNodes3) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build a b: cat c\n" +"build c: cat a\n")); + EXPECT_FALSE(plan_.AddTarget(GetNode("b"), &err)); + ASSERT_EQ("dependency cycle: c -> a -> c", err); +} + +TEST_F(PlanTest, CycleInEdgesButNotInNodes4) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build d: cat c\n" +"build c: cat b\n" +"build b: cat a\n" +"build a e: cat d\n" +"build f: cat e\n")); + EXPECT_FALSE(plan_.AddTarget(GetNode("f"), &err)); + ASSERT_EQ("dependency cycle: d -> c -> b -> a -> d", err); +} + void PlanTest::TestPoolWithDepthOne(const char* test_case) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case)); GetNode("out1")->MarkDirty(); @@ -456,8 +495,8 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { /// State of command_runner_ and logs contents (if specified) ARE MODIFIED. /// Handy to check for NOOP builds, and higher-level rebuild tests. void RebuildTarget(const string& target, const char* manifest, - const char* log_path = NULL, - const char* deps_path = NULL); + const char* log_path = NULL, const char* deps_path = NULL, + State* state = NULL); // Mark a path dirty. void Dirty(const string& path); @@ -477,10 +516,13 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { }; void BuildTest::RebuildTarget(const string& target, const char* manifest, - const char* log_path, const char* deps_path) { - State state; - ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); - AssertParse(&state, manifest); + const char* log_path, const char* deps_path, + State* state) { + State local_state, *pstate = &local_state; + if (state) + pstate = state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(pstate)); + AssertParse(pstate, manifest); string err; BuildLog build_log, *pbuild_log = NULL; @@ -493,20 +535,20 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, DepsLog deps_log, *pdeps_log = NULL; if (deps_path) { - ASSERT_TRUE(deps_log.Load(deps_path, &state, &err)); + ASSERT_TRUE(deps_log.Load(deps_path, pstate, &err)); ASSERT_TRUE(deps_log.OpenForWrite(deps_path, &err)); ASSERT_EQ("", err); pdeps_log = &deps_log; } - Builder builder(&state, config_, pbuild_log, pdeps_log, &fs_); + Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_); EXPECT_TRUE(builder.AddTarget(target, &err)); command_runner_.commands_ran_.clear(); builder.command_runner_.reset(&command_runner_); if (!builder.AlreadyUpToDate()) { bool build_res = builder.Build(&err); - EXPECT_TRUE(build_res) << "builder.Build(&err)"; + EXPECT_TRUE(build_res); } builder.command_runner_.release(); } @@ -676,7 +718,7 @@ TEST_F(BuildTest, TwoOutputs) { } // Test case from -// https://github.com/martine/ninja/issues/148 +// https://github.com/ninja-build/ninja/issues/148 TEST_F(BuildTest, MultiOutIn) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule touch\n" @@ -753,23 +795,18 @@ TEST_F(BuildTest, MakeDirs) { #ifdef _WIN32 ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n")); - EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err)); #else ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n")); - EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); #endif + EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); EXPECT_EQ("", err); EXPECT_TRUE(builder_.Build(&err)); ASSERT_EQ("", err); ASSERT_EQ(2u, fs_.directories_made_.size()); EXPECT_EQ("subdir", fs_.directories_made_[0]); -#ifdef _WIN32 - EXPECT_EQ("subdir\\dir2", fs_.directories_made_[1]); -#else EXPECT_EQ("subdir/dir2", fs_.directories_made_[1]); -#endif } TEST_F(BuildTest, DepFileMissing) { @@ -819,8 +856,7 @@ TEST_F(BuildTest, DepFileParseError) { fs_.Create("foo.c", ""); fs_.Create("foo.o.d", "randomtext\n"); EXPECT_FALSE(builder_.AddTarget("foo.o", &err)); - EXPECT_EQ("expected depfile 'foo.o.d' to mention 'foo.o', got 'randomtext'", - err); + EXPECT_EQ("foo.o.d: expected ':' in depfile", err); } TEST_F(BuildTest, OrderOnlyDeps) { @@ -940,6 +976,38 @@ TEST_F(BuildTest, RebuildOrderOnlyDeps) { ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]); } +#ifdef _WIN32 +TEST_F(BuildTest, DepFileCanonicalize) { + string err; + int orig_edges = state_.edges_.size(); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cc\n command = cc $in\n depfile = $out.d\n" +"build gen/stuff\\things/foo.o: cc x\\y/z\\foo.c\n")); + Edge* edge = state_.edges_.back(); + + fs_.Create("x/y/z/foo.c", ""); + GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. + // Note, different slashes from manifest. + fs_.Create("gen/stuff\\things/foo.o.d", + "gen\\stuff\\things\\foo.o: blah.h bar.h\n"); + EXPECT_TRUE(builder_.AddTarget("gen/stuff/things/foo.o", &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, fs_.files_read_.size()); + // The depfile path does not get Canonicalize as it seems unnecessary. + EXPECT_EQ("gen/stuff\\things/foo.o.d", fs_.files_read_[0]); + + // Expect three new edges: one generating foo.o, and two more from + // loading the depfile. + ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size()); + // Expect our edge to now have three inputs: foo.c and two headers. + ASSERT_EQ(3u, edge->inputs_.size()); + + // Expect the command line we generate to only use the original input, and + // using the slashes from the manifest. + ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand()); +} +#endif + TEST_F(BuildTest, Phony) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -1027,6 +1095,31 @@ TEST_F(BuildTest, SwallowFailuresLimit) { ASSERT_EQ("cannot make progress due to previous errors", err); } +TEST_F(BuildTest, PoolEdgesReadyButNotWanted) { + fs_.Create("x", ""); + + const char* manifest = + "pool some_pool\n" + " depth = 4\n" + "rule touch\n" + " command = touch $out\n" + " pool = some_pool\n" + "rule cc\n" + " command = touch grit\n" + "\n" + "build B.d.stamp: cc | x\n" + "build C.stamp: touch B.d.stamp\n" + "build final.stamp: touch || C.stamp\n"; + + RebuildTarget("final.stamp", manifest); + + fs_.RemoveFile("B.d.stamp"); + + State save_state; + RebuildTarget("final.stamp", manifest, NULL, NULL, &save_state); + EXPECT_GE(save_state.LookupPool("some_pool")->current_use(), 0); +} + struct BuildWithLogTest : public BuildTest { BuildWithLogTest() { builder_.SetBuildLog(&build_log_); @@ -1206,7 +1299,7 @@ TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) { } // Test scenario, in which an input file is removed, but output isn't changed -// https://github.com/martine/ninja/issues/295 +// https://github.com/ninja-build/ninja/issues/295 TEST_F(BuildWithLogTest, RestatMissingInput) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule true\n" @@ -1385,7 +1478,7 @@ TEST_F(BuildTest, RspFileFailure) { ASSERT_EQ("Another very long command", fs_.files_["out.rsp"].contents); } -// Test that contens of the RSP file behaves like a regular part of +// Test that contents of the RSP file behaves like a regular part of // command line, i.e. triggers a rebuild if changed TEST_F(BuildWithLogTest, RspFileCmdLineChange) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -1455,7 +1548,7 @@ TEST_F(BuildTest, InterruptCleanup) { EXPECT_FALSE(builder_.Build(&err)); EXPECT_EQ("interrupted by user", err); builder_.Cleanup(); - EXPECT_GT(fs_.Stat("out1"), 0); + EXPECT_GT(fs_.Stat("out1", &err), 0); err = ""; // A touched output of an interrupted command should be deleted. @@ -1464,7 +1557,22 @@ TEST_F(BuildTest, InterruptCleanup) { EXPECT_FALSE(builder_.Build(&err)); EXPECT_EQ("interrupted by user", err); builder_.Cleanup(); - EXPECT_EQ(0, fs_.Stat("out2")); + EXPECT_EQ(0, fs_.Stat("out2", &err)); +} + +TEST_F(BuildTest, StatFailureAbortsBuild) { + const string kTooLongToStat(400, 'i'); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +("build " + kTooLongToStat + ": cat " + kTooLongToStat + "\n").c_str())); + // Also cyclic, for good measure. + + // This simulates a stat failure: + fs_.files_[kTooLongToStat].mtime = -1; + fs_.files_[kTooLongToStat].stat_error = "stat failed"; + + string err; + EXPECT_FALSE(builder_.AddTarget(kTooLongToStat, &err)); + EXPECT_EQ("stat failed", err); } TEST_F(BuildTest, PhonyWithNoInputs) { @@ -1584,7 +1692,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { EXPECT_EQ("", err); // The deps file should have been removed. - EXPECT_EQ(0, fs_.Stat("in1.d")); + EXPECT_EQ(0, fs_.Stat("in1.d", &err)); // Recreate it for the next step. fs_.Create("in1.d", "out: in2"); deps_log.Close(); @@ -1664,7 +1772,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { fs_.Create("out", ""); // The deps file should have been removed, so no need to timestamp it. - EXPECT_EQ(0, fs_.Stat("in1.d")); + EXPECT_EQ(0, fs_.Stat("in1.d", &err)); { State state; @@ -1854,7 +1962,7 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { Edge* edge = state.edges_.back(); - state.GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. + state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); @@ -1872,8 +1980,74 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { } } +#ifdef _WIN32 +TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { + string err; + const char* manifest = + "rule cc\n command = cc $in\n depfile = $out.d\n deps = gcc\n" + "build a/b\\c\\d/e/fo$ o.o: cc x\\y/z\\foo.c\n"; + + fs_.Create("x/y/z/foo.c", ""); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + // Run the build once, everything should be ok. + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); + ASSERT_EQ("", err); + // Note, different slashes from manifest. + fs_.Create("a/b\\c\\d/e/fo o.o.d", + "a\\b\\c\\d\\e\\fo\\ o.o: blah.h bar.h\n"); + EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ("", err); + + deps_log.Close(); + builder.command_runner_.release(); + } + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_); + builder.command_runner_.reset(&command_runner_); + + Edge* edge = state.edges_.back(); + + state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. + EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); + ASSERT_EQ("", err); + + // Expect three new edges: one generating fo o.o, and two more from + // loading the depfile. + ASSERT_EQ(3u, state.edges_.size()); + // Expect our edge to now have three inputs: foo.c and two headers. + ASSERT_EQ(3u, edge->inputs_.size()); + + // Expect the command line we generate to only use the original input. + // Note, slashes from manifest, not .d. + ASSERT_EQ("cc x\\y/z\\foo.c", edge->EvaluateCommand()); + + deps_log.Close(); + builder.command_runner_.release(); + } +} +#endif + /// Check that a restat rule doesn't clear an edge if the depfile is missing. -/// Follows from: https://github.com/martine/ninja/issues/603 +/// Follows from: https://github.com/ninja-build/ninja/issues/603 TEST_F(BuildTest, RestatMissingDepfile) { const char* manifest = "rule true\n" @@ -1897,7 +2071,7 @@ const char* manifest = } /// Check that a restat rule doesn't clear an edge if the deps are missing. -/// https://github.com/martine/ninja/issues/603 +/// https://github.com/ninja-build/ninja/issues/603 TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { string err; const char* manifest = @@ -1948,6 +2122,23 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { ASSERT_EQ(0u, command_runner_.commands_ran_.size()); } +TEST_F(BuildTest, WrongOutputInDepfileCausesRebuild) { + string err; + const char* manifest = +"rule cc\n" +" command = cc $in\n" +" depfile = $out.d\n" +"build foo.o: cc foo.c\n"; + + fs_.Create("foo.c", ""); + fs_.Create("foo.o", ""); + fs_.Create("header.h", ""); + fs_.Create("foo.o.d", "bar.o.d: header.h\n"); + + RebuildTarget("foo.o", manifest, "build_log", "ninja_deps"); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); +} + TEST_F(BuildTest, Console) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule console\n" diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 59bd18f..389ac24 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -33,8 +33,9 @@ int main() { for (int j = 0; j < 5; ++j) { const int kNumRepetitions = 2000000; int64_t start = GetTimeMillis(); + unsigned int slash_bits; for (int i = 0; i < kNumRepetitions; ++i) { - CanonicalizePath(buf, &len, &err); + CanonicalizePath(buf, &len, &slash_bits, &err); } int delta = (int)(GetTimeMillis() - start); times.push_back(delta); diff --git a/src/clean.cc b/src/clean.cc index 98c638c..1d6ba9e 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -49,7 +49,11 @@ int Cleaner::RemoveFile(const string& path) { } bool Cleaner::FileExists(const string& path) { - return disk_interface_->Stat(path) > 0; + string err; + TimeStamp mtime = disk_interface_->Stat(path, &err); + if (mtime == -1) + Error("%s", err.c_str()); + return mtime > 0; // Treat Stat() errors as "file does not exist". } void Cleaner::Report(const string& path) { @@ -220,7 +224,7 @@ int Cleaner::CleanRule(const char* rule) { assert(rule); Reset(); - const Rule* r = state_->LookupRule(rule); + const Rule* r = state_->bindings_.LookupRule(rule); if (r) { CleanRule(r); } else { @@ -237,7 +241,7 @@ int Cleaner::CleanRules(int rule_count, char* rules[]) { PrintHeader(); for (int i = 0; i < rule_count; ++i) { const char* rule_name = rules[i]; - const Rule* rule = state_->LookupRule(rule_name); + const Rule* rule = state_->bindings_.LookupRule(rule_name); if (rule) { if (IsVerbose()) printf("Rule %s\n", rule_name); diff --git a/src/clean_test.cc b/src/clean_test.cc index 5869bbb..395343b 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -44,10 +44,11 @@ TEST_F(CleanTest, CleanAll) { EXPECT_EQ(4u, fs_.files_removed_.size()); // Check they are removed. - EXPECT_EQ(0, fs_.Stat("in1")); - EXPECT_EQ(0, fs_.Stat("out1")); - EXPECT_EQ(0, fs_.Stat("in2")); - EXPECT_EQ(0, fs_.Stat("out2")); + string err; + EXPECT_EQ(0, fs_.Stat("in1", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_EQ(0, fs_.Stat("in2", &err)); + EXPECT_EQ(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); EXPECT_EQ(0, cleaner.CleanAll()); @@ -75,10 +76,11 @@ TEST_F(CleanTest, CleanAllDryRun) { EXPECT_EQ(0u, fs_.files_removed_.size()); // Check they are not removed. - EXPECT_NE(0, fs_.Stat("in1")); - EXPECT_NE(0, fs_.Stat("out1")); - EXPECT_NE(0, fs_.Stat("in2")); - EXPECT_NE(0, fs_.Stat("out2")); + string err; + EXPECT_LT(0, fs_.Stat("in1", &err)); + EXPECT_LT(0, fs_.Stat("out1", &err)); + EXPECT_LT(0, fs_.Stat("in2", &err)); + EXPECT_LT(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); EXPECT_EQ(0, cleaner.CleanAll()); @@ -105,10 +107,11 @@ TEST_F(CleanTest, CleanTarget) { EXPECT_EQ(2u, fs_.files_removed_.size()); // Check they are removed. - EXPECT_EQ(0, fs_.Stat("in1")); - EXPECT_EQ(0, fs_.Stat("out1")); - EXPECT_NE(0, fs_.Stat("in2")); - EXPECT_NE(0, fs_.Stat("out2")); + string err; + EXPECT_EQ(0, fs_.Stat("in1", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_LT(0, fs_.Stat("in2", &err)); + EXPECT_LT(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); ASSERT_EQ(0, cleaner.CleanTarget("out1")); @@ -135,11 +138,12 @@ TEST_F(CleanTest, CleanTargetDryRun) { EXPECT_EQ(2, cleaner.cleaned_files_count()); EXPECT_EQ(0u, fs_.files_removed_.size()); - // Check they are removed. - EXPECT_NE(0, fs_.Stat("in1")); - EXPECT_NE(0, fs_.Stat("out1")); - EXPECT_NE(0, fs_.Stat("in2")); - EXPECT_NE(0, fs_.Stat("out2")); + // Check they are not removed. + string err; + EXPECT_LT(0, fs_.Stat("in1", &err)); + EXPECT_LT(0, fs_.Stat("out1", &err)); + EXPECT_LT(0, fs_.Stat("in2", &err)); + EXPECT_LT(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); ASSERT_EQ(0, cleaner.CleanTarget("out1")); @@ -168,10 +172,11 @@ TEST_F(CleanTest, CleanRule) { EXPECT_EQ(2u, fs_.files_removed_.size()); // Check they are removed. - EXPECT_EQ(0, fs_.Stat("in1")); - EXPECT_NE(0, fs_.Stat("out1")); - EXPECT_EQ(0, fs_.Stat("in2")); - EXPECT_NE(0, fs_.Stat("out2")); + string err; + EXPECT_EQ(0, fs_.Stat("in1", &err)); + EXPECT_LT(0, fs_.Stat("out1", &err)); + EXPECT_EQ(0, fs_.Stat("in2", &err)); + EXPECT_LT(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); ASSERT_EQ(0, cleaner.CleanRule("cat_e")); @@ -200,11 +205,12 @@ TEST_F(CleanTest, CleanRuleDryRun) { EXPECT_EQ(2, cleaner.cleaned_files_count()); EXPECT_EQ(0u, fs_.files_removed_.size()); - // Check they are removed. - EXPECT_NE(0, fs_.Stat("in1")); - EXPECT_NE(0, fs_.Stat("out1")); - EXPECT_NE(0, fs_.Stat("in2")); - EXPECT_NE(0, fs_.Stat("out2")); + // Check they are not removed. + string err; + EXPECT_LT(0, fs_.Stat("in1", &err)); + EXPECT_LT(0, fs_.Stat("out1", &err)); + EXPECT_LT(0, fs_.Stat("in2", &err)); + EXPECT_LT(0, fs_.Stat("out2", &err)); fs_.files_removed_.clear(); ASSERT_EQ(0, cleaner.CleanRule("cat_e")); @@ -328,12 +334,13 @@ TEST_F(CleanTest, CleanRsp) { EXPECT_EQ(6u, fs_.files_removed_.size()); // Check they are removed. - EXPECT_EQ(0, fs_.Stat("in1")); - EXPECT_EQ(0, fs_.Stat("out1")); - EXPECT_EQ(0, fs_.Stat("in2")); - EXPECT_EQ(0, fs_.Stat("out2")); - EXPECT_EQ(0, fs_.Stat("in2.rsp")); - EXPECT_EQ(0, fs_.Stat("out2.rsp")); + string err; + EXPECT_EQ(0, fs_.Stat("in1", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_EQ(0, fs_.Stat("in2", &err)); + EXPECT_EQ(0, fs_.Stat("out2", &err)); + EXPECT_EQ(0, fs_.Stat("in2.rsp", &err)); + EXPECT_EQ(0, fs_.Stat("out2.rsp", &err)); } TEST_F(CleanTest, CleanFailure) { @@ -345,6 +352,7 @@ TEST_F(CleanTest, CleanFailure) { } TEST_F(CleanTest, CleanPhony) { + string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build phony: phony t1 t2\n" "build t1: cat\n" @@ -358,7 +366,7 @@ TEST_F(CleanTest, CleanPhony) { Cleaner cleaner(&state_, config_, &fs_); EXPECT_EQ(0, cleaner.CleanAll()); EXPECT_EQ(2, cleaner.cleaned_files_count()); - EXPECT_NE(0, fs_.Stat("phony")); + EXPECT_LT(0, fs_.Stat("phony", &err)); fs_.Create("t1", ""); fs_.Create("t2", ""); @@ -366,7 +374,7 @@ TEST_F(CleanTest, CleanPhony) { // Check that CleanTarget does not remove "phony". EXPECT_EQ(0, cleaner.CleanTarget("phony")); EXPECT_EQ(2, cleaner.cleaned_files_count()); - EXPECT_NE(0, fs_.Stat("phony")); + EXPECT_LT(0, fs_.Stat("phony", &err)); } TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { @@ -391,8 +399,9 @@ TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { EXPECT_EQ(4, cleaner.cleaned_files_count()); EXPECT_EQ(4u, fs_.files_removed_.size()); - EXPECT_EQ(0, fs_.Stat("out 1")); - EXPECT_EQ(0, fs_.Stat("out 2")); - EXPECT_EQ(0, fs_.Stat("out 1.d")); - EXPECT_EQ(0, fs_.Stat("out 2.rsp")); + string err; + EXPECT_EQ(0, fs_.Stat("out 1", &err)); + EXPECT_EQ(0, fs_.Stat("out 2", &err)); + EXPECT_EQ(0, fs_.Stat("out 1.d", &err)); + EXPECT_EQ(0, fs_.Stat("out 2.rsp", &err)); } diff --git a/src/debug_flags.cc b/src/debug_flags.cc index 75f1ea5..8065001 100644 --- a/src/debug_flags.cc +++ b/src/debug_flags.cc @@ -15,3 +15,5 @@ bool g_explaining = false; bool g_keep_rsp = false; + +bool g_experimental_statcache = true; diff --git a/src/debug_flags.h b/src/debug_flags.h index ba3ebf3..7965585 100644 --- a/src/debug_flags.h +++ b/src/debug_flags.h @@ -26,4 +26,6 @@ extern bool g_explaining; extern bool g_keep_rsp; +extern bool g_experimental_statcache; + #endif // NINJA_EXPLAIN_H_ diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index dc53d4f..7cee892 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -234,5 +234,9 @@ yy16: return false; } } + if (parsing_targets) { + *err = "expected ':' in depfile"; + return false; + } return true; } diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 24744c7..98c1621 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -112,5 +112,9 @@ bool DepfileParser::Parse(string* content, string* err) { return false; } } + if (parsing_targets) { + *err = "expected ':' in depfile"; + return false; + } return true; } diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index c7a163c..ee798f8 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -14,7 +14,7 @@ #include "depfile_parser.h" -#include <gtest/gtest.h> +#include "test.h" struct DepfileParserTest : public testing::Test { bool Parse(const char* input, string* err); @@ -106,7 +106,7 @@ TEST_F(DepfileParserTest, Escapes) { // it through. string err; EXPECT_TRUE(Parse( -"\\!\\@\\#$$\\%\\^\\&\\\\", +"\\!\\@\\#$$\\%\\^\\&\\\\:", &err)); ASSERT_EQ("", err); EXPECT_EQ("\\!\\@#$\\%\\^\\&\\", @@ -142,8 +142,8 @@ TEST_F(DepfileParserTest, UnifyMultipleOutputs) { // check that multiple duplicate targets are properly unified string err; EXPECT_TRUE(Parse("foo foo: x y z", &err)); - ASSERT_EQ(parser_.out_.AsString(), "foo"); - ASSERT_EQ(parser_.ins_.size(), 3u); + ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); diff --git a/src/deps_log.cc b/src/deps_log.cc index 61df387..89c6023 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -239,8 +239,13 @@ bool DepsLog::Load(const string& path, State* state, string* err) { if (buf[path_size - 1] == '\0') --path_size; if (buf[path_size - 1] == '\0') --path_size; if (buf[path_size - 1] == '\0') --path_size; - StringPiece path(buf, path_size); - Node* node = state->GetNode(path); + StringPiece subpath(buf, path_size); + // It is not necessary to pass in a correct slash_bits here. It will + // either be a Node that's in the manifest (in which case it will already + // have a correct slash_bits that GetNode will look up), or it is an + // implicit dependency from a .d which does not affect the build command + // (and so need not have its slashes maintained). + Node* node = state->GetNode(subpath, 0); // Check that the expected index matches the actual index. This can only // happen if two ninja processes write to the same deps log concurrently. @@ -270,7 +275,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { } fclose(f); - if (!Truncate(path.c_str(), offset, err)) + if (!Truncate(path, offset, err)) return false; // The truncate succeeded; we'll just report the load error as a @@ -302,7 +307,6 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); - printf("Recompacting deps...\n"); Close(); string temp_path = path + ".recompact"; diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index e8e5138..cab06fb 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -14,6 +14,11 @@ #include "deps_log.h" +#include <sys/stat.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + #include "graph.h" #include "util.h" #include "test.h" @@ -41,16 +46,16 @@ TEST_F(DepsLogTest, WriteRead) { { vector<Node*> deps; - deps.push_back(state1.GetNode("foo.h")); - deps.push_back(state1.GetNode("bar.h")); - log1.RecordDeps(state1.GetNode("out.o"), 1, deps); + deps.push_back(state1.GetNode("foo.h", 0)); + deps.push_back(state1.GetNode("bar.h", 0)); + log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state1.GetNode("foo.h")); - deps.push_back(state1.GetNode("bar2.h")); - log1.RecordDeps(state1.GetNode("out2.o"), 2, deps); + deps.push_back(state1.GetNode("foo.h", 0)); + deps.push_back(state1.GetNode("bar2.h", 0)); + log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps); - DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o")); + DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); ASSERT_TRUE(log_deps); ASSERT_EQ(1, log_deps->mtime); ASSERT_EQ(2, log_deps->node_count); @@ -74,7 +79,7 @@ TEST_F(DepsLogTest, WriteRead) { } // Spot-check the entries in log2. - DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o")); + DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0)); ASSERT_TRUE(log_deps); ASSERT_EQ(2, log_deps->mtime); ASSERT_EQ(2, log_deps->node_count); @@ -96,11 +101,11 @@ TEST_F(DepsLogTest, LotsOfDeps) { for (int i = 0; i < kNumDeps; ++i) { char buf[32]; sprintf(buf, "file%d.h", i); - deps.push_back(state1.GetNode(buf)); + deps.push_back(state1.GetNode(buf, 0)); } - log1.RecordDeps(state1.GetNode("out.o"), 1, deps); + log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps); - DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o")); + DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0)); ASSERT_EQ(kNumDeps, log_deps->node_count); } @@ -111,7 +116,7 @@ TEST_F(DepsLogTest, LotsOfDeps) { EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err)); ASSERT_EQ("", err); - DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o")); + DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0)); ASSERT_EQ(kNumDeps, log_deps->node_count); } @@ -127,9 +132,9 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -149,9 +154,9 @@ TEST_F(DepsLogTest, DoubleEntry) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -181,14 +186,14 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("baz.h")); - log.RecordDeps(state.GetNode("other_out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("baz.h", 0)); + log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps); log.Close(); @@ -211,8 +216,8 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); struct stat st; @@ -232,14 +237,14 @@ TEST_F(DepsLogTest, Recompact) { string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); - Node* out = state.GetNode("out.o"); + Node* out = state.GetNode("out.o", 0); DepsLog::Deps* deps = log.GetDeps(out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); ASSERT_EQ(1, deps->node_count); ASSERT_EQ("foo.h", deps->nodes[0]->path()); - Node* other_out = state.GetNode("other_out.o"); + Node* other_out = state.GetNode("other_out.o", 0); deps = log.GetDeps(other_out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); @@ -281,14 +286,14 @@ TEST_F(DepsLogTest, Recompact) { string err; ASSERT_TRUE(log.Load(kTestFilename, &state, &err)); - Node* out = state.GetNode("out.o"); + Node* out = state.GetNode("out.o", 0); DepsLog::Deps* deps = log.GetDeps(out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); ASSERT_EQ(1, deps->node_count); ASSERT_EQ("foo.h", deps->nodes[0]->path()); - Node* other_out = state.GetNode("other_out.o"); + Node* other_out = state.GetNode("other_out.o", 0); deps = log.GetDeps(other_out); ASSERT_TRUE(deps); ASSERT_EQ(1, deps->mtime); @@ -354,14 +359,14 @@ TEST_F(DepsLogTest, Truncated) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 2, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); log.Close(); } @@ -413,14 +418,14 @@ TEST_F(DepsLogTest, TruncatedRecovery) { ASSERT_EQ("", err); vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar.h")); - log.RecordDeps(state.GetNode("out.o"), 1, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); deps.clear(); - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 2, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); log.Close(); } @@ -441,16 +446,16 @@ TEST_F(DepsLogTest, TruncatedRecovery) { err.clear(); // The truncated entry should've been discarded. - EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o"))); + EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0))); EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); ASSERT_EQ("", err); // Add a new entry. vector<Node*> deps; - deps.push_back(state.GetNode("foo.h")); - deps.push_back(state.GetNode("bar2.h")); - log.RecordDeps(state.GetNode("out2.o"), 3, deps); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 3, deps); log.Close(); } @@ -464,7 +469,7 @@ TEST_F(DepsLogTest, TruncatedRecovery) { EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); // The truncated entry should exist. - DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o")); + DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0)); ASSERT_TRUE(deps); } } diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 4dfae1a..70282c0 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -23,6 +23,7 @@ #include <sys/types.h> #ifdef _WIN32 +#include <sstream> #include <windows.h> #include <direct.h> // _mkdir #endif @@ -55,6 +56,76 @@ int MakeDir(const string& path) { #endif } +#ifdef _WIN32 +TimeStamp TimeStampFromFileTime(const FILETIME& filetime) { + // FILETIME is in 100-nanosecond increments since the Windows epoch. + // We don't much care about epoch correctness but we do want the + // resulting value to fit in an integer. + uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | + ((uint64_t)filetime.dwLowDateTime); + mtime /= 1000000000LL / 100; // 100ns -> s. + mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). + return (TimeStamp)mtime; +} + +TimeStamp StatSingleFile(const string& path, string* err) { + WIN32_FILE_ATTRIBUTE_DATA attrs; + if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) + return 0; + *err = "GetFileAttributesEx(" + path + "): " + GetLastErrorString(); + return -1; + } + return TimeStampFromFileTime(attrs.ftLastWriteTime); +} + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4996) // GetVersionExA is deprecated post SDK 8.1. +#endif +bool IsWindows7OrLater() { + OSVERSIONINFO version_info = { sizeof(version_info) }; + if (!GetVersionEx(&version_info)) + Fatal("GetVersionEx: %s", GetLastErrorString().c_str()); + return version_info.dwMajorVersion > 6 || + (version_info.dwMajorVersion == 6 && version_info.dwMinorVersion >= 1); +} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps, + string* err) { + // FindExInfoBasic is 30% faster than FindExInfoStandard. + static bool can_use_basic_info = IsWindows7OrLater(); + // This is not in earlier SDKs. + const FINDEX_INFO_LEVELS kFindExInfoBasic = + static_cast<FINDEX_INFO_LEVELS>(1); + FINDEX_INFO_LEVELS level = + can_use_basic_info ? kFindExInfoBasic : FindExInfoStandard; + WIN32_FIND_DATAA ffd; + HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), level, &ffd, + FindExSearchNameMatch, NULL, 0); + + if (find_handle == INVALID_HANDLE_VALUE) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) + return true; + *err = "FindFirstFileExA(" + dir + "): " + GetLastErrorString(); + return false; + } + do { + string lowername = ffd.cFileName; + transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower); + stamps->insert(make_pair(lowername, + TimeStampFromFileTime(ffd.ftLastWriteTime))); + } while (FindNextFileA(find_handle, &ffd)); + FindClose(find_handle); + return true; +} +#endif // _WIN32 + } // namespace // DiskInterface --------------------------------------------------------------- @@ -63,9 +134,12 @@ bool DiskInterface::MakeDirs(const string& path) { string dir = DirName(path); if (dir.empty()) return true; // Reached root; assume it's there. - TimeStamp mtime = Stat(dir); - if (mtime < 0) - return false; // Error. + string err; + TimeStamp mtime = Stat(dir, &err); + if (mtime < 0) { + Error("%s", err.c_str()); + return false; + } if (mtime > 0) return true; // Exists already; we're done. @@ -78,45 +152,42 @@ bool DiskInterface::MakeDirs(const string& path) { // RealDiskInterface ----------------------------------------------------------- -TimeStamp RealDiskInterface::Stat(const string& path) { +TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { #ifdef _WIN32 // MSDN: "Naming Files, Paths, and Namespaces" // http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) { - if (!quiet_) { - Error("Stat(%s): Filename longer than %i characters", - path.c_str(), MAX_PATH); - } + ostringstream err_stream; + err_stream << "Stat(" << path << "): Filename longer than " << MAX_PATH + << " characters"; + *err = err_stream.str(); return -1; } - WIN32_FILE_ATTRIBUTE_DATA attrs; - if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) { - DWORD err = GetLastError(); - if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) - return 0; - if (!quiet_) { - Error("GetFileAttributesEx(%s): %s", path.c_str(), - GetLastErrorString().c_str()); + if (!use_cache_) + return StatSingleFile(path, err); + + string dir = DirName(path); + string base(path.substr(dir.size() ? dir.size() + 1 : 0)); + + transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + transform(base.begin(), base.end(), base.begin(), ::tolower); + + Cache::iterator ci = cache_.find(dir); + if (ci == cache_.end()) { + ci = cache_.insert(make_pair(dir, DirCache())).first; + if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, err)) { + cache_.erase(ci); + return -1; } - return -1; } - const FILETIME& filetime = attrs.ftLastWriteTime; - // FILETIME is in 100-nanosecond increments since the Windows epoch. - // We don't much care about epoch correctness but we do want the - // resulting value to fit in an integer. - uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) | - ((uint64_t)filetime.dwLowDateTime); - mtime /= 1000000000LL / 100; // 100ns -> s. - mtime -= 12622770400LL; // 1600 epoch -> 2000 epoch (subtract 400 years). - return (TimeStamp)mtime; + DirCache::iterator di = ci->second.find(base); + return di != ci->second.end() ? di->second : 0; #else struct stat st; if (stat(path.c_str(), &st) < 0) { if (errno == ENOENT || errno == ENOTDIR) return 0; - if (!quiet_) { - Error("stat(%s): %s", path.c_str(), strerror(errno)); - } + *err = "stat(" + path + "): " + strerror(errno); return -1; } return st.st_mtime; @@ -181,3 +252,11 @@ int RealDiskInterface::RemoveFile(const string& path) { return 0; } } + +void RealDiskInterface::AllowStatCache(bool allow) { +#ifdef _WIN32 + use_cache_ = allow; + if (!use_cache_) + cache_.clear(); +#endif +} diff --git a/src/disk_interface.h b/src/disk_interface.h index ff1e21c..b61d192 100644 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -15,6 +15,7 @@ #ifndef NINJA_DISK_INTERFACE_H_ #define NINJA_DISK_INTERFACE_H_ +#include <map> #include <string> using namespace std; @@ -29,7 +30,7 @@ struct DiskInterface { /// stat() a file, returning the mtime, or 0 if missing and -1 on /// other errors. - virtual TimeStamp Stat(const string& path) = 0; + virtual TimeStamp Stat(const string& path, string* err) const = 0; /// Create a directory, returning false on failure. virtual bool MakeDir(const string& path) = 0; @@ -55,16 +56,32 @@ struct DiskInterface { /// Implementation of DiskInterface that actually hits the disk. struct RealDiskInterface : public DiskInterface { - RealDiskInterface() : quiet_(false) {} + RealDiskInterface() +#ifdef _WIN32 + : use_cache_(false) +#endif + {} virtual ~RealDiskInterface() {} - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path, string* err) const; virtual bool MakeDir(const string& path); virtual bool WriteFile(const string& path, const string& contents); virtual string ReadFile(const string& path, string* err); virtual int RemoveFile(const string& path); - /// Whether to print on errors. Used to make a test quieter. - bool quiet_; + /// Whether stat information can be cached. Only has an effect on Windows. + void AllowStatCache(bool allow); + + private: +#ifdef _WIN32 + /// Whether stat information can be cached. + bool use_cache_; + + typedef map<string, TimeStamp> DirCache; + // TODO: Neither a map nor a hashmap seems ideal here. If the statcache + // works out, come up with a better data structure. + typedef map<string, DirCache> Cache; + mutable Cache cache_; +#endif }; #endif // NINJA_DISK_INTERFACE_H_ diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 51a1d14..9d210b4 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <gtest/gtest.h> - +#include <assert.h> +#include <stdio.h> #ifdef _WIN32 #include <io.h> #include <windows.h> @@ -47,35 +47,114 @@ struct DiskInterfaceTest : public testing::Test { }; TEST_F(DiskInterfaceTest, StatMissingFile) { - EXPECT_EQ(0, disk_.Stat("nosuchfile")); + string err; + EXPECT_EQ(0, disk_.Stat("nosuchfile", &err)); + EXPECT_EQ("", err); // On Windows, the errno for a file in a nonexistent directory // is different. - EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile")); + EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile", &err)); + EXPECT_EQ("", err); // On POSIX systems, the errno is different if a component of the // path prefix is not a directory. ASSERT_TRUE(Touch("notadir")); - EXPECT_EQ(0, disk_.Stat("notadir/nosuchfile")); + EXPECT_EQ(0, disk_.Stat("notadir/nosuchfile", &err)); + EXPECT_EQ("", err); } TEST_F(DiskInterfaceTest, StatBadPath) { - disk_.quiet_ = true; + string err; #ifdef _WIN32 string bad_path("cc:\\foo"); - EXPECT_EQ(-1, disk_.Stat(bad_path)); + EXPECT_EQ(-1, disk_.Stat(bad_path, &err)); + EXPECT_NE("", err); #else string too_long_name(512, 'x'); - EXPECT_EQ(-1, disk_.Stat(too_long_name)); + EXPECT_EQ(-1, disk_.Stat(too_long_name, &err)); + EXPECT_NE("", err); #endif - disk_.quiet_ = false; } TEST_F(DiskInterfaceTest, StatExistingFile) { + string err; ASSERT_TRUE(Touch("file")); - EXPECT_GT(disk_.Stat("file"), 1); + EXPECT_GT(disk_.Stat("file", &err), 1); + EXPECT_EQ("", err); +} + +TEST_F(DiskInterfaceTest, StatExistingDir) { + string err; + ASSERT_TRUE(disk_.MakeDir("subdir")); + ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + EXPECT_GT(disk_.Stat(".", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("subdir", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1); + EXPECT_EQ("", err); + + EXPECT_EQ(disk_.Stat("subdir", &err), + disk_.Stat("subdir/.", &err)); + EXPECT_EQ(disk_.Stat("subdir", &err), + disk_.Stat("subdir/subsubdir/..", &err)); + EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err), + disk_.Stat("subdir/subsubdir/.", &err)); } +#ifdef _WIN32 +TEST_F(DiskInterfaceTest, StatCache) { + string err; + disk_.AllowStatCache(true); + + ASSERT_TRUE(Touch("file1")); + ASSERT_TRUE(Touch("fiLE2")); + ASSERT_TRUE(disk_.MakeDir("subdir")); + ASSERT_TRUE(disk_.MakeDir("subdir/subsubdir")); + ASSERT_TRUE(Touch("subdir\\subfile1")); + ASSERT_TRUE(Touch("subdir\\SUBFILE2")); + ASSERT_TRUE(Touch("subdir\\SUBFILE3")); + + EXPECT_GT(disk_.Stat("FIle1", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("file1", &err), 1); + EXPECT_EQ("", err); + + EXPECT_GT(disk_.Stat("subdir/subfile2", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("sUbdir\\suBFile1", &err), 1); + EXPECT_EQ("", err); + + EXPECT_GT(disk_.Stat(".", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("subdir", &err), 1); + EXPECT_EQ("", err); + EXPECT_GT(disk_.Stat("subdir/subsubdir", &err), 1); + EXPECT_EQ("", err); + + EXPECT_EQ(disk_.Stat("subdir", &err), + disk_.Stat("subdir/.", &err)); + EXPECT_EQ("", err); + EXPECT_EQ(disk_.Stat("subdir", &err), + disk_.Stat("subdir/subsubdir/..", &err)); + EXPECT_EQ("", err); + EXPECT_EQ(disk_.Stat("subdir/subsubdir", &err), + disk_.Stat("subdir/subsubdir/.", &err)); + EXPECT_EQ("", err); + + // Test error cases. + string bad_path("cc:\\foo"); + EXPECT_EQ(-1, disk_.Stat(bad_path, &err)); + EXPECT_NE("", err); err.clear(); + EXPECT_EQ(-1, disk_.Stat(bad_path, &err)); + EXPECT_NE("", err); err.clear(); + EXPECT_EQ(0, disk_.Stat("nosuchfile", &err)); + EXPECT_EQ("", err); + EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile", &err)); + EXPECT_EQ("", err); +} +#endif + TEST_F(DiskInterfaceTest, ReadFile) { string err; EXPECT_EQ("", disk_.ReadFile("foobar", &err)); @@ -120,7 +199,7 @@ struct StatTest : public StateTestWithBuiltinRules, StatTest() : scan_(&state_, NULL, NULL, this) {} // DiskInterface implementation. - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path, string* err) const; virtual bool WriteFile(const string& path, const string& contents) { assert(false); return true; @@ -140,12 +219,12 @@ struct StatTest : public StateTestWithBuiltinRules, DependencyScan scan_; map<string, TimeStamp> mtimes_; - vector<string> stats_; + mutable vector<string> stats_; }; -TimeStamp StatTest::Stat(const string& path) { +TimeStamp StatTest::Stat(const string& path, string* err) const { stats_.push_back(path); - map<string, TimeStamp>::iterator i = mtimes_.find(path); + map<string, TimeStamp>::const_iterator i = mtimes_.find(path); if (i == mtimes_.end()) return 0; // File not found. return i->second; @@ -156,7 +235,9 @@ TEST_F(StatTest, Simple) { "build out: cat in\n")); Node* out = GetNode("out"); - out->Stat(this); + string err; + EXPECT_TRUE(out->Stat(this, &err)); + EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(2u, stats_.size()); @@ -170,7 +251,9 @@ TEST_F(StatTest, TwoStep) { "build mid: cat in\n")); Node* out = GetNode("out"); - out->Stat(this); + string err; + EXPECT_TRUE(out->Stat(this, &err)); + EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(3u, stats_.size()); @@ -188,7 +271,9 @@ TEST_F(StatTest, Tree) { "build mid2: cat in21 in22\n")); Node* out = GetNode("out"); - out->Stat(this); + string err; + EXPECT_TRUE(out->Stat(this, &err)); + EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_EQ(1u + 6u, stats_.size()); @@ -207,7 +292,9 @@ TEST_F(StatTest, Middle) { mtimes_["out"] = 1; Node* out = GetNode("out"); - out->Stat(this); + string err; + EXPECT_TRUE(out->Stat(this, &err)); + EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); scan_.RecomputeDirty(out->in_edge(), NULL); ASSERT_FALSE(GetNode("in")->dirty()); diff --git a/src/edit_distance.cc b/src/edit_distance.cc index 9553c6e..3bb62b8 100644 --- a/src/edit_distance.cc +++ b/src/edit_distance.cc @@ -28,40 +28,42 @@ int EditDistance(const StringPiece& s1, // http://en.wikipedia.org/wiki/Levenshtein_distance // // Although the algorithm is typically described using an m x n - // array, only two rows are used at a time, so this implemenation - // just keeps two separate vectors for those two rows. + // array, only one row plus one element are used at a time, so this + // implementation just keeps one vector for the row. To update one entry, + // only the entries to the left, top, and top-left are needed. The left + // entry is in row[x-1], the top entry is what's in row[x] from the last + // iteration, and the top-left entry is stored in previous. int m = s1.len_; int n = s2.len_; - vector<int> previous(n + 1); - vector<int> current(n + 1); - - for (int i = 0; i <= n; ++i) - previous[i] = i; + vector<int> row(n + 1); + for (int i = 1; i <= n; ++i) + row[i] = i; for (int y = 1; y <= m; ++y) { - current[0] = y; - int best_this_row = current[0]; + row[0] = y; + int best_this_row = row[0]; + int previous = y - 1; for (int x = 1; x <= n; ++x) { + int old_row = row[x]; if (allow_replacements) { - current[x] = min(previous[x-1] + (s1.str_[y-1] == s2.str_[x-1] ? 0 : 1), - min(current[x-1], previous[x])+1); + row[x] = min(previous + (s1.str_[y - 1] == s2.str_[x - 1] ? 0 : 1), + min(row[x - 1], row[x]) + 1); } else { - if (s1.str_[y-1] == s2.str_[x-1]) - current[x] = previous[x-1]; + if (s1.str_[y - 1] == s2.str_[x - 1]) + row[x] = previous; else - current[x] = min(current[x-1], previous[x]) + 1; + row[x] = min(row[x - 1], row[x]) + 1; } - best_this_row = min(best_this_row, current[x]); + previous = old_row; + best_this_row = min(best_this_row, row[x]); } if (max_edit_distance && best_this_row > max_edit_distance) return max_edit_distance + 1; - - current.swap(previous); } - return previous[n]; + return row[n]; } diff --git a/src/eval_env.cc b/src/eval_env.cc index 834b7e1..e991d21 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <assert.h> + #include "eval_env.h" string BindingEnv::LookupVariable(const string& var) { @@ -27,6 +29,55 @@ void BindingEnv::AddBinding(const string& key, const string& val) { bindings_[key] = val; } +void BindingEnv::AddRule(const Rule* rule) { + assert(LookupRuleCurrentScope(rule->name()) == NULL); + rules_[rule->name()] = rule; +} + +const Rule* BindingEnv::LookupRuleCurrentScope(const string& rule_name) { + map<string, const Rule*>::iterator i = rules_.find(rule_name); + if (i == rules_.end()) + return NULL; + return i->second; +} + +const Rule* BindingEnv::LookupRule(const string& rule_name) { + map<string, const Rule*>::iterator i = rules_.find(rule_name); + if (i != rules_.end()) + return i->second; + if (parent_) + return parent_->LookupRule(rule_name); + return NULL; +} + +void Rule::AddBinding(const string& key, const EvalString& val) { + bindings_[key] = val; +} + +const EvalString* Rule::GetBinding(const string& key) const { + map<string, EvalString>::const_iterator i = bindings_.find(key); + if (i == bindings_.end()) + return NULL; + return &i->second; +} + +// static +bool Rule::IsReservedBinding(const string& var) { + return var == "command" || + var == "depfile" || + var == "description" || + var == "deps" || + var == "generator" || + var == "pool" || + var == "restat" || + var == "rspfile" || + var == "rspfile_content"; +} + +const map<string, const Rule*>& BindingEnv::GetRules() const { + return rules_; +} + string BindingEnv::LookupWithFallback(const string& var, const EvalString* eval, Env* env) { diff --git a/src/eval_env.h b/src/eval_env.h index f3c959a..28c4d16 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -22,7 +22,7 @@ using namespace std; #include "string_piece.h" -struct EvalString; +struct Rule; /// An interface for a scope for variable (e.g. "$foo") lookups. struct Env { @@ -30,15 +30,62 @@ struct Env { virtual string LookupVariable(const string& var) = 0; }; +/// A tokenized string that contains variable references. +/// Can be evaluated relative to an Env. +struct EvalString { + string Evaluate(Env* env) const; + + void Clear() { parsed_.clear(); } + bool empty() const { return parsed_.empty(); } + + void AddText(StringPiece text); + void AddSpecial(StringPiece text); + + /// Construct a human-readable representation of the parsed state, + /// for use in tests. + string Serialize() const; + +private: + enum TokenType { RAW, SPECIAL }; + typedef vector<pair<string, TokenType> > TokenList; + TokenList parsed_; +}; + +/// An invokable build command and associated metadata (description, etc.). +struct Rule { + explicit Rule(const string& name) : name_(name) {} + + const string& name() const { return name_; } + + typedef map<string, EvalString> Bindings; + void AddBinding(const string& key, const EvalString& val); + + static bool IsReservedBinding(const string& var); + + const EvalString* GetBinding(const string& key) const; + + private: + // Allow the parsers to reach into this object and fill out its fields. + friend struct ManifestParser; + + string name_; + map<string, EvalString> bindings_; +}; + /// An Env which contains a mapping of variables to values /// as well as a pointer to a parent scope. struct BindingEnv : public Env { BindingEnv() : parent_(NULL) {} - explicit BindingEnv(Env* parent) : parent_(parent) {} + explicit BindingEnv(BindingEnv* parent) : parent_(parent) {} virtual ~BindingEnv() {} virtual string LookupVariable(const string& var); + void AddRule(const Rule* rule); + const Rule* LookupRule(const string& rule_name); + const Rule* LookupRuleCurrentScope(const string& rule_name); + const map<string, const Rule*>& GetRules() const; + void AddBinding(const string& key, const string& val); /// This is tricky. Edges want lookup scope to go in this order: @@ -51,28 +98,8 @@ struct BindingEnv : public Env { private: map<string, string> bindings_; - Env* parent_; -}; - -/// A tokenized string that contains variable references. -/// Can be evaluated relative to an Env. -struct EvalString { - string Evaluate(Env* env) const; - - void Clear() { parsed_.clear(); } - bool empty() const { return parsed_.empty(); } - - void AddText(StringPiece text); - void AddSpecial(StringPiece text); - - /// Construct a human-readable representation of the parsed state, - /// for use in tests. - string Serialize() const; - -private: - enum TokenType { RAW, SPECIAL }; - typedef vector<pair<string, TokenType> > TokenList; - TokenList parsed_; + map<string, const Rule*> rules_; + BindingEnv* parent_; }; #endif // NINJA_EVAL_ENV_H_ diff --git a/src/getopt.c b/src/getopt.c index 75ef99c..3350fb9 100644 --- a/src/getopt.c +++ b/src/getopt.c @@ -299,7 +299,7 @@ getopt_internal (int argc, char **argv, char *shortopts, return (optopt = '?'); } has_arg = ((cp[1] == ':') - ? ((cp[2] == ':') ? OPTIONAL_ARG : REQUIRED_ARG) : no_argument); + ? ((cp[2] == ':') ? OPTIONAL_ARG : required_argument) : no_argument); possible_arg = argv[optind] + optwhere + 1; optopt = *cp; } @@ -318,7 +318,7 @@ getopt_internal (int argc, char **argv, char *shortopts, else optarg = NULL; break; - case REQUIRED_ARG: + case required_argument: if (*possible_arg == '=') possible_arg++; if (*possible_arg != '\0') diff --git a/src/getopt.h b/src/getopt.h index ead9878..b4247fb 100644 --- a/src/getopt.h +++ b/src/getopt.h @@ -4,9 +4,9 @@ /* include files needed by this include file */ /* macros defined by this include file */ -#define no_argument 0 -#define REQUIRED_ARG 1 -#define OPTIONAL_ARG 2 +#define no_argument 0 +#define required_argument 1 +#define OPTIONAL_ARG 2 /* types defined by this include file */ diff --git a/src/graph.cc b/src/graph.cc index aa9c0e8..9e65675 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -27,34 +27,9 @@ #include "state.h" #include "util.h" -bool Node::Stat(DiskInterface* disk_interface) { +bool Node::Stat(DiskInterface* disk_interface, string* err) { METRIC_RECORD("node stat"); - mtime_ = disk_interface->Stat(path_); - return mtime_ > 0; -} - -void Rule::AddBinding(const string& key, const EvalString& val) { - bindings_[key] = val; -} - -const EvalString* Rule::GetBinding(const string& key) const { - map<string, EvalString>::const_iterator i = bindings_.find(key); - if (i == bindings_.end()) - return NULL; - return &i->second; -} - -// static -bool Rule::IsReservedBinding(const string& var) { - return var == "command" || - var == "depfile" || - var == "description" || - var == "deps" || - var == "generator" || - var == "pool" || - var == "restat" || - var == "rspfile" || - var == "rspfile_content"; + return (mtime_ = disk_interface->Stat(path_, err)) != -1; } bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { @@ -62,10 +37,25 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { edge->outputs_ready_ = true; edge->deps_missing_ = false; + // RecomputeDirty() recursively walks the graph following the input nodes + // of |edge| and the in_edges of these nodes. It uses the stat state of each + // node to mark nodes as visited and doesn't traverse across nodes that have + // been visited already. To make sure that every edge is visited only once + // (important because an edge's deps are loaded every time it's visited), mark + // all outputs of |edge| visited as a first step. This ensures that edges + // with multiple inputs and outputs are visited only once, even in cyclic + // graphs. + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + if (!(*o)->StatIfNecessary(disk_interface_, err)) + return false; + } + if (!dep_loader_.LoadDeps(edge, err)) { if (!err->empty()) return false; // Failed to load dependency info: rebuild to regenerate it. + // LoadDeps() did EXPLAIN() already, no need to do it here. dirty = edge->deps_missing_ = true; } @@ -73,7 +63,9 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { Node* most_recent_input = NULL; for (vector<Node*>::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { - if ((*i)->StatIfNecessary(disk_interface_)) { + if (!(*i)->status_known()) { + if (!(*i)->StatIfNecessary(disk_interface_, err)) + return false; if (Edge* in_edge = (*i)->in_edge()) { if (!RecomputeDirty(in_edge, err)) return false; @@ -108,15 +100,14 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { // We may also be dirty due to output state: missing outputs, out of // date outputs, etc. Visit all outputs and determine whether they're dirty. if (!dirty) - dirty = RecomputeOutputsDirty(edge, most_recent_input); + if (!RecomputeOutputsDirty(edge, most_recent_input, &dirty, err)) + return false; - // Finally, visit each output to mark off that we've visited it, and update - // their dirty state if necessary. - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface_); + // Finally, visit each output and update their dirty state if necessary. + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { if (dirty) - (*i)->MarkDirty(); + (*o)->MarkDirty(); } // If an edge is dirty, its outputs are normally not ready. (It's @@ -130,16 +121,19 @@ bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { return true; } -bool DependencyScan::RecomputeOutputsDirty(Edge* edge, - Node* most_recent_input) { - string command = edge->EvaluateCommand(true); - for (vector<Node*>::iterator i = edge->outputs_.begin(); - i != edge->outputs_.end(); ++i) { - (*i)->StatIfNecessary(disk_interface_); - if (RecomputeOutputDirty(edge, most_recent_input, command, *i)) +bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, + bool* outputs_dirty, string* err) { + string command = edge->EvaluateCommand(/*incl_rsp_file=*/true); + for (vector<Node*>::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + if (!(*o)->StatIfNecessary(disk_interface_, err)) + return false; + if (RecomputeOutputDirty(edge, most_recent_input, command, *o)) { + *outputs_dirty = true; return true; + } } - return false; + return true; } bool DependencyScan::RecomputeOutputDirty(Edge* edge, @@ -149,7 +143,12 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, if (edge->is_phony()) { // Phony edges don't write any output. Outputs are only dirty if // there are no inputs and we're missing the output. - return edge->inputs_.empty() && !output->exists(); + if (edge->inputs_.empty() && !output->exists()) { + EXPLAIN("output %s of phony edge with no inputs doesn't exist", + output->path().c_str()); + return true; + } + return false; } BuildLog::LogEntry* entry = 0; @@ -217,8 +216,8 @@ bool Edge::AllInputsReady() const { struct EdgeEnv : public Env { enum EscapeKind { kShellEscape, kDoNotEscape }; - explicit EdgeEnv(Edge* edge, EscapeKind escape) - : edge_(edge), escape_in_out_(escape) {} + EdgeEnv(Edge* edge, EscapeKind escape) + : edge_(edge), escape_in_out_(escape), recursive_(false) {} virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command @@ -227,8 +226,11 @@ struct EdgeEnv : public Env { vector<Node*>::iterator end, char sep); + private: + vector<string> lookups_; Edge* edge_; EscapeKind escape_in_out_; + bool recursive_; }; string EdgeEnv::LookupVariable(const string& var) { @@ -244,8 +246,25 @@ string EdgeEnv::LookupVariable(const string& var) { ' '); } + if (recursive_) { + vector<string>::const_iterator it; + if ((it = find(lookups_.begin(), lookups_.end(), var)) != lookups_.end()) { + string cycle; + for (; it != lookups_.end(); ++it) + cycle.append(*it + " -> "); + cycle.append(var); + Fatal(("cycle in rule variables: " + cycle).c_str()); + } + } + // See notes on BindingEnv::LookupWithFallback. const EvalString* eval = edge_->rule_->GetBinding(var); + if (recursive_ && eval) + lookups_.push_back(var); + + // In practice, variables defined on rules never use another rule variable. + // For performance, only start checking for cycles after the first lookup. + recursive_ = true; return edge_->env_->LookupWithFallback(var, eval, this); } @@ -256,7 +275,7 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin, for (vector<Node*>::iterator i = begin; i != end; ++i) { if (!result.empty()) result.push_back(sep); - const string& path = (*i)->path(); + const string& path = (*i)->PathDecanonicalized(); if (escape_in_out_ == kShellEscape) { #if _WIN32 GetWin32EscapedString(path, &result); @@ -328,6 +347,21 @@ bool Edge::use_console() const { return pool() == &State::kConsolePool; } +// static +string Node::PathDecanonicalized(const string& path, unsigned int slash_bits) { + string result = path; +#ifdef _WIN32 + unsigned int mask = 1; + for (char* c = &result[0]; (c = strchr(c, '/')) != NULL;) { + if (slash_bits & mask) + *c = '\\'; + c++; + mask <<= 1; + } +#endif + return result; +} + void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", prefix, path().c_str(), this, @@ -379,12 +413,18 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } - // Check that this depfile matches the edge's output. + unsigned int unused; + if (!CanonicalizePath(const_cast<char*>(depfile.out_.str_), + &depfile.out_.len_, &unused, err)) + return false; + + // Check that this depfile matches the edge's output, if not return false to + // mark the edge as dirty. Node* first_output = edge->outputs_[0]; StringPiece opath = StringPiece(first_output->path()); if (opath != depfile.out_) { - *err = "expected depfile '" + path + "' to mention '" + - first_output->path() + "', got '" + depfile.out_.AsString() + "'"; + EXPLAIN("expected depfile '%s' to mention '%s', got '%s'", path.c_str(), + first_output->path().c_str(), depfile.out_.AsString().c_str()); return false; } @@ -395,10 +435,12 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, // Add all its in-edges. for (vector<StringPiece>::iterator i = depfile.ins_.begin(); i != depfile.ins_.end(); ++i, ++implicit_dep) { - if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, err)) + unsigned int slash_bits; + if (!CanonicalizePath(const_cast<char*>(i->str_), &i->len_, &slash_bits, + err)) return false; - Node* node = state_->GetNode(*i); + Node* node = state_->GetNode(*i, slash_bits); *implicit_dep = node; node->AddOutEdge(edge); CreatePhonyInEdge(node); @@ -418,7 +460,7 @@ bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { // Deps are invalid if the output is newer than the deps. if (output->mtime() > deps->mtime) { - EXPLAIN("stored deps info out of date for for '%s' (%d vs %d)", + EXPLAIN("stored deps info out of date for '%s' (%d vs %d)", output->path().c_str(), deps->mtime, output->mtime()); return false; } diff --git a/src/graph.h b/src/graph.h index 66e31b5..cf15123 100644 --- a/src/graph.h +++ b/src/graph.h @@ -33,22 +33,22 @@ struct State; /// Information about a node in the dependency graph: the file, whether /// it's dirty, mtime, etc. struct Node { - explicit Node(const string& path) + Node(const string& path, unsigned int slash_bits) : path_(path), + slash_bits_(slash_bits), mtime_(-1), dirty_(false), in_edge_(NULL), id_(-1) {} - /// Return true if the file exists (mtime_ got a value). - bool Stat(DiskInterface* disk_interface); + /// Return false on error. + bool Stat(DiskInterface* disk_interface, string* err); - /// Return true if we needed to stat. - bool StatIfNecessary(DiskInterface* disk_interface) { + /// Return false on error. + bool StatIfNecessary(DiskInterface* disk_interface, string* err) { if (status_known()) - return false; - Stat(disk_interface); - return true; + return true; + return Stat(disk_interface, err); } /// Mark as not-yet-stat()ed and not dirty. @@ -71,6 +71,14 @@ struct Node { } const string& path() const { return path_; } + /// Get |path()| but use slash_bits to convert back to original slash styles. + string PathDecanonicalized() const { + return PathDecanonicalized(path_, slash_bits_); + } + static string PathDecanonicalized(const string& path, + unsigned int slash_bits); + unsigned int slash_bits() const { return slash_bits_; } + TimeStamp mtime() const { return mtime_; } bool dirty() const { return dirty_; } @@ -90,6 +98,11 @@ struct Node { private: string path_; + + /// Set bits starting from lowest for backslashes that were normalized to + /// forward slashes by CanonicalizePath. See |PathDecanonicalized|. + unsigned int slash_bits_; + /// Possible values of mtime_: /// -1: file hasn't been examined /// 0: we looked, and file doesn't exist @@ -112,30 +125,10 @@ private: int id_; }; -/// An invokable build command and associated metadata (description, etc.). -struct Rule { - explicit Rule(const string& name) : name_(name) {} - - const string& name() const { return name_; } - - typedef map<string, EvalString> Bindings; - void AddBinding(const string& key, const EvalString& val); - - static bool IsReservedBinding(const string& var); - - const EvalString* GetBinding(const string& key) const; - - private: - // Allow the parsers to reach into this object and fill out its fields. - friend struct ManifestParser; - - string name_; - map<string, EvalString> bindings_; -}; - /// An edge in the dependency graph; links between Nodes using Rules. struct Edge { - Edge() : rule_(NULL), env_(NULL), outputs_ready_(false), deps_missing_(false), + Edge() : rule_(NULL), pool_(NULL), env_(NULL), + outputs_ready_(false), deps_missing_(false), implicit_deps_(0), order_only_deps_(0) {} /// Return true if all inputs' in-edges are ready. @@ -248,9 +241,10 @@ struct DependencyScan { /// Returns false on failure. bool RecomputeDirty(Edge* edge, string* err); - /// Recompute whether any output of the edge is dirty. - /// Returns true if so. - bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input); + /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. + /// Returns false on failure. + bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, + bool* dirty, string* err); BuildLog* build_log() const { return build_log_; diff --git a/src/graph_test.cc b/src/graph_test.cc index 14dc678..44be8a5 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -153,7 +153,7 @@ TEST_F(GraphTest, VarInOutPathEscaping) { #endif } -// Regression test for https://github.com/martine/ninja/issues/380 +// Regression test for https://github.com/ninja-build/ninja/issues/380 TEST_F(GraphTest, DepfileWithCanonicalizablePath) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule catdep\n" @@ -172,7 +172,7 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { EXPECT_FALSE(GetNode("out.o")->dirty()); } -// Regression test for https://github.com/martine/ninja/issues/404 +// Regression test for https://github.com/ninja-build/ninja/issues/404 TEST_F(GraphTest, DepfileRemoved) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule catdep\n" @@ -251,3 +251,99 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { EXPECT_EQ(0, plan_.command_edge_count()); ASSERT_FALSE(plan_.more_to_do()); } + +// Verify that cycles in graphs with multiple outputs are handled correctly +// in RecomputeDirty() and don't cause deps to be loaded multiple times. +TEST_F(GraphTest, CycleWithLengthZeroFromDepfile) { + AssertParse(&state_, +"rule deprule\n" +" depfile = dep.d\n" +" command = unused\n" +"build a b: deprule\n" + ); + fs_.Create("dep.d", "a: b\n"); + + string err; + Edge* edge = GetNode("a")->in_edge(); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + + // Despite the depfile causing edge to be a cycle (it has outputs a and b, + // but the depfile also adds b as an input), the deps should have been loaded + // only once: + EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ("b", edge->inputs_[0]->path()); +} + +// Like CycleWithLengthZeroFromDepfile but with a higher cycle length. +TEST_F(GraphTest, CycleWithLengthOneFromDepfile) { + AssertParse(&state_, +"rule deprule\n" +" depfile = dep.d\n" +" command = unused\n" +"rule r\n" +" command = unused\n" +"build a b: deprule\n" +"build c: r b\n" + ); + fs_.Create("dep.d", "a: c\n"); + + string err; + Edge* edge = GetNode("a")->in_edge(); + EXPECT_TRUE(scan_.RecomputeDirty(edge, &err)); + ASSERT_EQ("", err); + + // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, + // but c's in_edge has b as input but the depfile also adds |edge| as + // output)), the deps should have been loaded only once: + EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ("c", edge->inputs_[0]->path()); +} + +// Like CycleWithLengthOneFromDepfile but building a node one hop away from +// the cycle. +TEST_F(GraphTest, CycleWithLengthOneFromDepfileOneHopAway) { + AssertParse(&state_, +"rule deprule\n" +" depfile = dep.d\n" +" command = unused\n" +"rule r\n" +" command = unused\n" +"build a b: deprule\n" +"build c: r b\n" +"build d: r a\n" + ); + fs_.Create("dep.d", "a: c\n"); + + string err; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("d")->in_edge(), &err)); + ASSERT_EQ("", err); + + // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, + // but c's in_edge has b as input but the depfile also adds |edge| as + // output)), the deps should have been loaded only once: + Edge* edge = GetNode("a")->in_edge(); + EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ("c", edge->inputs_[0]->path()); +} + +#ifdef _WIN32 +TEST_F(GraphTest, Decanonicalize) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out\\out1: cat src\\in1\n" +"build out\\out2/out3\\out4: cat mid1\n" +"build out3 out4\\foo: cat mid1\n")); + + string err; + vector<Node*> root_nodes = state_.RootNodes(&err); + EXPECT_EQ(4u, root_nodes.size()); + EXPECT_EQ(root_nodes[0]->path(), "out/out1"); + EXPECT_EQ(root_nodes[1]->path(), "out/out2/out3/out4"); + EXPECT_EQ(root_nodes[2]->path(), "out3"); + EXPECT_EQ(root_nodes[3]->path(), "out4/foo"); + EXPECT_EQ(root_nodes[0]->PathDecanonicalized(), "out\\out1"); + EXPECT_EQ(root_nodes[1]->PathDecanonicalized(), "out\\out2/out3\\out4"); + EXPECT_EQ(root_nodes[2]->PathDecanonicalized(), "out3"); + EXPECT_EQ(root_nodes[3]->PathDecanonicalized(), "out4\\foo"); +} +#endif diff --git a/src/graphviz.cc b/src/graphviz.cc index 8354a22..dce8b32 100644 --- a/src/graphviz.cc +++ b/src/graphviz.cc @@ -15,6 +15,7 @@ #include "graphviz.h" #include <stdio.h> +#include <algorithm> #include "graph.h" @@ -22,7 +23,9 @@ void GraphViz::AddTarget(Node* node) { if (visited_nodes_.find(node) != visited_nodes_.end()) return; - printf("\"%p\" [label=\"%s\"]\n", node, node->path().c_str()); + string pathstr = node->path(); + replace(pathstr.begin(), pathstr.end(), '\\', '/'); + printf("\"%p\" [label=\"%s\"]\n", node, pathstr.c_str()); visited_nodes_.insert(node); Edge* edge = node->in_edge(); diff --git a/src/graphviz.h b/src/graphviz.h index 1e2a29d..408496d 100644 --- a/src/graphviz.h +++ b/src/graphviz.h @@ -16,7 +16,6 @@ #define NINJA_GRAPHVIZ_H_ #include <set> -using namespace std; struct Node; struct Edge; @@ -27,8 +26,8 @@ struct GraphViz { void AddTarget(Node* node); void Finish(); - set<Node*> visited_nodes_; - set<Edge*> visited_edges_; + std::set<Node*> visited_nodes_; + std::set<Edge*> visited_edges_; }; #endif // NINJA_GRAPHVIZ_H_ diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index d0eabde..5be0531 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -46,7 +46,7 @@ int main() { sort(hashes, hashes + N); - int num_collisions = 0; + int collision_count = 0; for (int i = 1; i < N; ++i) { if (hashes[i - 1].first == hashes[i].first) { if (strcmp(commands[hashes[i - 1].second], @@ -54,9 +54,9 @@ int main() { printf("collision!\n string 1: '%s'\n string 2: '%s'\n", commands[hashes[i - 1].second], commands[hashes[i].second]); - num_collisions++; + collision_count++; } } } - printf("\n\n%d collisions after %d runs\n", num_collisions, N); + printf("\n\n%d collisions after %d runs\n", collision_count, N); } diff --git a/src/hash_map.h b/src/hash_map.h index 77e7586..a91aeb9 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -50,7 +50,22 @@ unsigned int MurmurHash2(const void* key, size_t len) { return h; } -#ifdef _MSC_VER +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) +#include <unordered_map> + +namespace std { +template<> +struct hash<StringPiece> { + typedef StringPiece argument_type; + typedef size_t result_type; + + size_t operator()(StringPiece key) const { + return MurmurHash2(key.str_, key.len_); + } +}; +} + +#elif defined(_MSC_VER) #include <hash_map> using stdext::hash_map; @@ -61,7 +76,7 @@ struct StringPieceCmp : public hash_compare<StringPiece> { return MurmurHash2(key.str_, key.len_); } bool operator()(const StringPiece& a, const StringPiece& b) const { - int cmp = strncmp(a.str_, b.str_, min(a.len_, b.len_)); + int cmp = memcmp(a.str_, b.str_, min(a.len_, b.len_)); if (cmp < 0) { return true; } else if (cmp > 0) { @@ -73,26 +88,17 @@ struct StringPieceCmp : public hash_compare<StringPiece> { }; #else - #include <ext/hash_map> using __gnu_cxx::hash_map; namespace __gnu_cxx { template<> -struct hash<std::string> { - size_t operator()(const std::string& s) const { - return hash<const char*>()(s.c_str()); - } -}; - -template<> struct hash<StringPiece> { size_t operator()(StringPiece key) const { return MurmurHash2(key.str_, key.len_); } }; - } #endif @@ -102,7 +108,9 @@ struct hash<StringPiece> { /// mapping StringPiece => Foo*. template<typename V> struct ExternalStringHashMap { -#ifdef _MSC_VER +#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) + typedef std::unordered_map<StringPiece, V> Type; +#elif defined(_MSC_VER) typedef hash_map<StringPiece, V, StringPieceCmp> Type; #else typedef hash_map<StringPiece, V> Type; diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 05ce75d..ca35012 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -68,12 +68,15 @@ string IncludesNormalize::ToLower(const string& s) { string IncludesNormalize::AbsPath(StringPiece s) { char result[_MAX_PATH]; GetFullPathName(s.AsString().c_str(), sizeof(result), result, NULL); + for (char* c = result; *c; ++c) + if (*c == '\\') + *c = '/'; return result; } string IncludesNormalize::Relativize(StringPiece path, const string& start) { - vector<string> start_list = Split(AbsPath(start), '\\'); - vector<string> path_list = Split(AbsPath(path), '\\'); + vector<string> start_list = Split(AbsPath(start), '/'); + vector<string> path_list = Split(AbsPath(path), '/'); int i; for (i = 0; i < static_cast<int>(min(start_list.size(), path_list.size())); ++i) { @@ -88,28 +91,32 @@ string IncludesNormalize::Relativize(StringPiece path, const string& start) { rel_list.push_back(path_list[j]); if (rel_list.size() == 0) return "."; - return Join(rel_list, '\\'); + return Join(rel_list, '/'); } -string IncludesNormalize::Normalize(const string& input, - const char* relative_to) { - char copy[_MAX_PATH]; +bool IncludesNormalize::Normalize(const string& input, const char* relative_to, + string* result, string* err) { + char copy[_MAX_PATH + 1]; size_t len = input.size(); - strncpy(copy, input.c_str(), input.size() + 1); - for (size_t j = 0; j < len; ++j) - if (copy[j] == '/') - copy[j] = '\\'; - string err; - if (!CanonicalizePath(copy, &len, &err)) { - Warning("couldn't canonicalize '%s: %s\n", input.c_str(), err.c_str()); + if (len > _MAX_PATH) { + *err = "path too long"; + return false; } + strncpy(copy, input.c_str(), input.size() + 1); + unsigned int slash_bits; + if (!CanonicalizePath(copy, &len, &slash_bits, err)) + return false; + StringPiece partially_fixed(copy, len); + string curdir; if (!relative_to) { curdir = AbsPath("."); relative_to = curdir.c_str(); } - StringPiece partially_fixed(copy, len); - if (!SameDrive(partially_fixed, relative_to)) - return partially_fixed.AsString(); - return Relativize(partially_fixed, relative_to); + if (!SameDrive(partially_fixed, relative_to)) { + *result = partially_fixed.AsString(); + return true; + } + *result = Relativize(partially_fixed, relative_to); + return true; } diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 43527af..98e912f 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -29,7 +29,7 @@ struct IncludesNormalize { static string Relativize(StringPiece path, const string& start); /// Normalize by fixing slashes style, fixing redundant .. and . and makes the - /// path relative to |relative_to|. Case is normalized to lowercase on - /// Windows too. - static string Normalize(const string& input, const char* relative_to); + /// path relative to |relative_to|. + static bool Normalize(const string& input, const char* relative_to, + string* result, string* err); }; diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index 419996f..aba25d0 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -14,18 +14,13 @@ #include "includes_normalize.h" -#include <gtest/gtest.h> +#include <algorithm> + +#include <direct.h> #include "test.h" #include "util.h" -TEST(IncludesNormalize, Simple) { - EXPECT_EQ("b", IncludesNormalize::Normalize("a\\..\\b", NULL)); - EXPECT_EQ("b", IncludesNormalize::Normalize("a\\../b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); -} - namespace { string GetCurDir() { @@ -35,28 +30,50 @@ string GetCurDir() { return parts[parts.size() - 1]; } +string NormalizeAndCheckNoError(const std::string& input) { + string result, err; + EXPECT_TRUE(IncludesNormalize::Normalize(input.c_str(), NULL, &result, &err)); + EXPECT_EQ("", err); + return result; +} + +string NormalizeRelativeAndCheckNoError(const std::string& input, + const std::string& relative_to) { + string result, err; + EXPECT_TRUE(IncludesNormalize::Normalize(input.c_str(), relative_to.c_str(), + &result, &err)); + EXPECT_EQ("", err); + return result; +} + } // namespace +TEST(IncludesNormalize, Simple) { + EXPECT_EQ("b", NormalizeAndCheckNoError("a\\..\\b")); + EXPECT_EQ("b", NormalizeAndCheckNoError("a\\../b")); + EXPECT_EQ("a/b", NormalizeAndCheckNoError("a\\.\\b")); + EXPECT_EQ("a/b", NormalizeAndCheckNoError("a\\./b")); +} + TEST(IncludesNormalize, WithRelative) { string currentdir = GetCurDir(); - EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b")); - EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), - NULL)); - EXPECT_EQ(string("..\\") + currentdir + string("\\a"), - IncludesNormalize::Normalize("a", "../b")); - EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"), - IncludesNormalize::Normalize("a/b", "../c")); - EXPECT_EQ("..\\..\\a", IncludesNormalize::Normalize("a", "b/c")); - EXPECT_EQ(".", IncludesNormalize::Normalize("a", "a")); + EXPECT_EQ("c", NormalizeRelativeAndCheckNoError("a/b/c", "a/b")); + EXPECT_EQ("a", NormalizeAndCheckNoError(IncludesNormalize::AbsPath("a"))); + EXPECT_EQ(string("../") + currentdir + string("/a"), + NormalizeRelativeAndCheckNoError("a", "../b")); + EXPECT_EQ(string("../") + currentdir + string("/a/b"), + NormalizeRelativeAndCheckNoError("a/b", "../c")); + EXPECT_EQ("../../a", NormalizeRelativeAndCheckNoError("a", "b/c")); + EXPECT_EQ(".", NormalizeRelativeAndCheckNoError("a", "a")); } TEST(IncludesNormalize, Case) { - EXPECT_EQ("b", IncludesNormalize::Normalize("Abc\\..\\b", NULL)); - EXPECT_EQ("BdEf", IncludesNormalize::Normalize("Abc\\..\\BdEf", NULL)); - EXPECT_EQ("A\\b", IncludesNormalize::Normalize("A\\.\\b", NULL)); - EXPECT_EQ("a\\b", IncludesNormalize::Normalize("a\\./b", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\.\\B", NULL)); - EXPECT_EQ("A\\B", IncludesNormalize::Normalize("A\\./B", NULL)); + EXPECT_EQ("b", NormalizeAndCheckNoError("Abc\\..\\b")); + EXPECT_EQ("BdEf", NormalizeAndCheckNoError("Abc\\..\\BdEf")); + EXPECT_EQ("A/b", NormalizeAndCheckNoError("A\\.\\b")); + EXPECT_EQ("a/b", NormalizeAndCheckNoError("a\\./b")); + EXPECT_EQ("A/B", NormalizeAndCheckNoError("A\\.\\B")); + EXPECT_EQ("A/B", NormalizeAndCheckNoError("A\\./B")); } TEST(IncludesNormalize, Join) { @@ -89,16 +106,63 @@ TEST(IncludesNormalize, ToLower) { TEST(IncludesNormalize, DifferentDrive) { EXPECT_EQ("stuff.h", - IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "p:\\vs08")); + NormalizeRelativeAndCheckNoError("p:\\vs08\\stuff.h", "p:\\vs08")); EXPECT_EQ("stuff.h", - IncludesNormalize::Normalize("P:\\Vs08\\stuff.h", "p:\\vs08")); - EXPECT_EQ("p:\\vs08\\stuff.h", - IncludesNormalize::Normalize("p:\\vs08\\stuff.h", "c:\\vs08")); - EXPECT_EQ("P:\\vs08\\stufF.h", - IncludesNormalize::Normalize("P:\\vs08\\stufF.h", "D:\\stuff/things")); - EXPECT_EQ("P:\\vs08\\stuff.h", - IncludesNormalize::Normalize("P:/vs08\\stuff.h", "D:\\stuff/things")); - // TODO: this fails; fix it. - //EXPECT_EQ("P:\\wee\\stuff.h", - // IncludesNormalize::Normalize("P:/vs08\\../wee\\stuff.h", "D:\\stuff/things")); + NormalizeRelativeAndCheckNoError("P:\\Vs08\\stuff.h", "p:\\vs08")); + EXPECT_EQ("p:/vs08/stuff.h", + NormalizeRelativeAndCheckNoError("p:\\vs08\\stuff.h", "c:\\vs08")); + EXPECT_EQ("P:/vs08/stufF.h", NormalizeRelativeAndCheckNoError( + "P:\\vs08\\stufF.h", "D:\\stuff/things")); + EXPECT_EQ("P:/vs08/stuff.h", NormalizeRelativeAndCheckNoError( + "P:/vs08\\stuff.h", "D:\\stuff/things")); + EXPECT_EQ("P:/wee/stuff.h", + NormalizeRelativeAndCheckNoError("P:/vs08\\../wee\\stuff.h", + "D:\\stuff/things")); +} + +TEST(IncludesNormalize, LongInvalidPath) { + const char kLongInputString[] = + "C:\\Program Files (x86)\\Microsoft Visual Studio " + "12.0\\VC\\INCLUDEwarning #31001: The dll for reading and writing the " + "pdb (for example, mspdb110.dll) could not be found on your path. This " + "is usually a configuration error. Compilation will continue using /Z7 " + "instead of /Zi, but expect a similar error when you link your program."; + // Too long, won't be canonicalized. Ensure doesn't crash. + string result, err; + EXPECT_FALSE( + IncludesNormalize::Normalize(kLongInputString, NULL, &result, &err)); + EXPECT_EQ("path too long", err); + + const char kExactlyMaxPath[] = + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "012345678\\" + "0123456789"; + std::string forward_slashes(kExactlyMaxPath); + std::replace(forward_slashes.begin(), forward_slashes.end(), '\\', '/'); + // Make sure a path that's exactly _MAX_PATH long is canonicalized. + EXPECT_EQ(forward_slashes, + NormalizeAndCheckNoError(kExactlyMaxPath)); } diff --git a/src/lexer.cc b/src/lexer.cc index 685fe81..37b8678 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -103,8 +103,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -129,7 +127,7 @@ Lexer::Token Lexer::ReadToken() { unsigned int yyaccept = 0; static const unsigned char yybm[] = { 0, 64, 64, 64, 64, 64, 64, 64, - 64, 64, 0, 64, 64, 0, 64, 64, + 64, 64, 0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 192, 64, 64, 64, 64, 64, 64, 64, @@ -163,56 +161,66 @@ Lexer::Token Lexer::ReadToken() { }; yych = *p; - if (yych <= '^') { - if (yych <= ',') { - if (yych <= 0x1F) { - if (yych <= 0x00) goto yy22; - if (yych == '\n') goto yy6; - goto yy24; + if (yych <= 'Z') { + if (yych <= '#') { + if (yych <= '\f') { + if (yych <= 0x00) goto yy23; + if (yych == '\n') goto yy7; + goto yy25; } else { - if (yych <= ' ') goto yy2; - if (yych == '#') goto yy4; - goto yy24; + if (yych <= 0x1F) { + if (yych <= '\r') goto yy6; + goto yy25; + } else { + if (yych <= ' ') goto yy2; + if (yych <= '"') goto yy25; + goto yy4; + } } } else { - if (yych <= ':') { - if (yych == '/') goto yy24; - if (yych <= '9') goto yy21; - goto yy15; + if (yych <= '9') { + if (yych <= ',') goto yy25; + if (yych == '/') goto yy25; + goto yy22; } else { - if (yych <= '=') { - if (yych <= '<') goto yy24; - goto yy13; + if (yych <= '<') { + if (yych <= ':') goto yy16; + goto yy25; } else { - if (yych <= '@') goto yy24; - if (yych <= 'Z') goto yy21; - goto yy24; + if (yych <= '=') goto yy14; + if (yych <= '@') goto yy25; + goto yy22; } } } } else { if (yych <= 'i') { - if (yych <= 'b') { - if (yych == '`') goto yy24; - if (yych <= 'a') goto yy21; - goto yy8; + if (yych <= 'a') { + if (yych == '_') goto yy22; + if (yych <= '`') goto yy25; + goto yy22; } else { - if (yych == 'd') goto yy12; - if (yych <= 'h') goto yy21; - goto yy19; + if (yych <= 'c') { + if (yych <= 'b') goto yy9; + goto yy22; + } else { + if (yych <= 'd') goto yy13; + if (yych <= 'h') goto yy22; + goto yy20; + } } } else { if (yych <= 'r') { - if (yych == 'p') goto yy10; - if (yych <= 'q') goto yy21; - goto yy11; + if (yych == 'p') goto yy11; + if (yych <= 'q') goto yy22; + goto yy12; } else { if (yych <= 'z') { - if (yych <= 's') goto yy20; - goto yy21; + if (yych <= 's') goto yy21; + goto yy22; } else { - if (yych == '|') goto yy17; - goto yy24; + if (yych == '|') goto yy18; + goto yy25; } } } @@ -220,192 +228,203 @@ Lexer::Token Lexer::ReadToken() { yy2: yyaccept = 0; yych = *(q = ++p); - goto yy70; + goto yy73; yy3: { token = INDENT; break; } yy4: yyaccept = 1; yych = *(q = ++p); - if (yych <= 0x00) goto yy5; - if (yych != '\r') goto yy65; + if (yych >= 0x01) goto yy68; yy5: { token = ERROR; break; } yy6: - ++p; + yych = *++p; + if (yych == '\n') goto yy65; + goto yy5; yy7: - { token = NEWLINE; break; } -yy8: ++p; - if ((yych = *p) == 'u') goto yy59; - goto yy26; +yy8: + { token = NEWLINE; break; } yy9: - { token = IDENT; break; } + ++p; + if ((yych = *p) == 'u') goto yy60; + goto yy27; yy10: - yych = *++p; - if (yych == 'o') goto yy55; - goto yy26; + { token = IDENT; break; } yy11: yych = *++p; - if (yych == 'u') goto yy51; - goto yy26; + if (yych == 'o') goto yy56; + goto yy27; yy12: yych = *++p; - if (yych == 'e') goto yy44; - goto yy26; + if (yych == 'u') goto yy52; + goto yy27; yy13: + yych = *++p; + if (yych == 'e') goto yy45; + goto yy27; +yy14: ++p; { token = EQUALS; break; } -yy15: +yy16: ++p; { token = COLON; break; } -yy17: +yy18: ++p; - if ((yych = *p) == '|') goto yy42; + if ((yych = *p) == '|') goto yy43; { token = PIPE; break; } -yy19: - yych = *++p; - if (yych == 'n') goto yy35; - goto yy26; yy20: yych = *++p; - if (yych == 'u') goto yy27; - goto yy26; + if (yych == 'n') goto yy36; + goto yy27; yy21: yych = *++p; - goto yy26; + if (yych == 'u') goto yy28; + goto yy27; yy22: + yych = *++p; + goto yy27; +yy23: ++p; { token = TEOF; break; } -yy24: +yy25: yych = *++p; goto yy5; -yy25: +yy26: ++p; yych = *p; -yy26: +yy27: if (yybm[0+yych] & 32) { - goto yy25; + goto yy26; } - goto yy9; -yy27: + goto yy10; +yy28: yych = *++p; - if (yych != 'b') goto yy26; + if (yych != 'b') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'n') goto yy26; + if (yych != 'n') goto yy27; yych = *++p; - if (yych != 'j') goto yy26; + if (yych != 'j') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = SUBNINJA; break; } -yy35: +yy36: yych = *++p; - if (yych != 'c') goto yy26; + if (yych != 'c') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = INCLUDE; break; } -yy42: +yy43: ++p; { token = PIPE2; break; } -yy44: +yy45: yych = *++p; - if (yych != 'f') goto yy26; + if (yych != 'f') goto yy27; yych = *++p; - if (yych != 'a') goto yy26; + if (yych != 'a') goto yy27; yych = *++p; - if (yych != 'u') goto yy26; + if (yych != 'u') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 't') goto yy26; + if (yych != 't') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = DEFAULT; break; } -yy51: +yy52: yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'e') goto yy26; + if (yych != 'e') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = RULE; break; } -yy55: +yy56: yych = *++p; - if (yych != 'o') goto yy26; + if (yych != 'o') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = POOL; break; } -yy59: +yy60: yych = *++p; - if (yych != 'i') goto yy26; + if (yych != 'i') goto yy27; yych = *++p; - if (yych != 'l') goto yy26; + if (yych != 'l') goto yy27; yych = *++p; - if (yych != 'd') goto yy26; + if (yych != 'd') goto yy27; ++p; if (yybm[0+(yych = *p)] & 32) { - goto yy25; + goto yy26; } { token = BUILD; break; } -yy64: +yy65: + ++p; + { token = NEWLINE; break; } +yy67: ++p; yych = *p; -yy65: +yy68: if (yybm[0+yych] & 64) { - goto yy64; + goto yy67; } - if (yych <= 0x00) goto yy66; - if (yych <= '\f') goto yy67; -yy66: + if (yych >= 0x01) goto yy70; +yy69: p = q; if (yyaccept <= 0) { goto yy3; } else { goto yy5; } -yy67: +yy70: ++p; { continue; } -yy69: +yy72: yyaccept = 0; q = ++p; yych = *p; -yy70: +yy73: if (yybm[0+yych] & 128) { - goto yy69; + goto yy72; + } + if (yych <= '\f') { + if (yych != '\n') goto yy3; + } else { + if (yych <= '\r') goto yy75; + if (yych == '#') goto yy67; + goto yy3; } - if (yych == '\n') goto yy71; - if (yych == '#') goto yy64; - goto yy3; -yy71: + yych = *++p; + goto yy8; +yy75: ++p; - yych = *p; - goto yy7; + if ((yych = *p) == '\n') goto yy65; + goto yy69; } } @@ -427,6 +446,7 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; @@ -468,39 +488,48 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yych <= ' ') { - if (yych <= 0x00) goto yy78; - if (yych <= 0x1F) goto yy80; + if (yych <= 0x00) goto yy82; + if (yych <= 0x1F) goto yy84; } else { - if (yych == '$') goto yy76; - goto yy80; + if (yych == '$') goto yy80; + goto yy84; } ++p; yych = *p; - goto yy84; -yy75: + goto yy92; +yy79: { continue; } -yy76: - ++p; - if ((yych = *p) == '\n') goto yy81; -yy77: +yy80: + yych = *(q = ++p); + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; +yy81: { break; } -yy78: +yy82: ++p; { break; } -yy80: +yy84: yych = *++p; - goto yy77; -yy81: + goto yy81; +yy85: ++p; { continue; } -yy83: +yy87: + yych = *++p; + if (yych == '\n') goto yy89; + p = q; + goto yy81; +yy89: + ++p; + { continue; } +yy91: ++p; yych = *p; -yy84: +yy92: if (yybm[0+yych] & 128) { - goto yy83; + goto yy91; } - goto yy75; + goto yy79; } } @@ -550,40 +579,40 @@ bool Lexer::ReadIdent(string* out) { yych = *p; if (yych <= '@') { if (yych <= '.') { - if (yych <= ',') goto yy89; + if (yych <= ',') goto yy97; } else { - if (yych <= '/') goto yy89; - if (yych >= ':') goto yy89; + if (yych <= '/') goto yy97; + if (yych >= ':') goto yy97; } } else { if (yych <= '_') { - if (yych <= 'Z') goto yy87; - if (yych <= '^') goto yy89; + if (yych <= 'Z') goto yy95; + if (yych <= '^') goto yy97; } else { - if (yych <= '`') goto yy89; - if (yych >= '{') goto yy89; + if (yych <= '`') goto yy97; + if (yych >= '{') goto yy97; } } -yy87: +yy95: ++p; yych = *p; - goto yy92; -yy88: + goto yy100; +yy96: { out->assign(start, p - start); break; } -yy89: +yy97: ++p; { return false; } -yy91: +yy99: ++p; yych = *p; -yy92: +yy100: if (yybm[0+yych] & 128) { - goto yy91; + goto yy99; } - goto yy88; + goto yy96; } } @@ -638,29 +667,36 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { yych = *p; if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy101; - if (yych >= '\n') goto yy97; + if (yych <= 0x00) goto yy110; + if (yych >= '\n') goto yy107; } else { - if (yych == '\r') goto yy103; - if (yych >= ' ') goto yy97; + if (yych == '\r') goto yy105; + if (yych >= ' ') goto yy107; } } else { if (yych <= '9') { - if (yych == '$') goto yy99; + if (yych == '$') goto yy109; } else { - if (yych <= ':') goto yy97; - if (yych == '|') goto yy97; + if (yych <= ':') goto yy107; + if (yych == '|') goto yy107; } } ++p; yych = *p; - goto yy126; -yy96: + goto yy140; +yy104: { eval->AddText(StringPiece(start, p - start)); continue; } -yy97: +yy105: + ++p; + if ((yych = *p) == '\n') goto yy137; + { + last_token_ = start; + return Error(DescribeLastError(), err); + } +yy107: ++p; { if (path) { @@ -673,137 +709,152 @@ yy97: continue; } } -yy99: - ++p; - if ((yych = *p) <= '/') { - if (yych <= ' ') { - if (yych == '\n') goto yy115; - if (yych <= 0x1F) goto yy104; - goto yy106; +yy109: + yych = *++p; + if (yych <= '-') { + if (yych <= 0x1F) { + if (yych <= '\n') { + if (yych <= '\t') goto yy112; + goto yy124; + } else { + if (yych == '\r') goto yy114; + goto yy112; + } } else { - if (yych <= '$') { - if (yych <= '#') goto yy104; - goto yy108; + if (yych <= '#') { + if (yych <= ' ') goto yy115; + goto yy112; } else { - if (yych == '-') goto yy110; - goto yy104; + if (yych <= '$') goto yy117; + if (yych <= ',') goto yy112; + goto yy119; } } } else { - if (yych <= '^') { - if (yych <= ':') { - if (yych <= '9') goto yy110; - goto yy112; + if (yych <= 'Z') { + if (yych <= '9') { + if (yych <= '/') goto yy112; + goto yy119; } else { - if (yych <= '@') goto yy104; - if (yych <= 'Z') goto yy110; - goto yy104; + if (yych <= ':') goto yy121; + if (yych <= '@') goto yy112; + goto yy119; } } else { if (yych <= '`') { - if (yych <= '_') goto yy110; - goto yy104; + if (yych == '_') goto yy119; + goto yy112; } else { - if (yych <= 'z') goto yy110; - if (yych <= '{') goto yy114; - goto yy104; + if (yych <= 'z') goto yy119; + if (yych <= '{') goto yy123; + goto yy112; } } } -yy100: - { - last_token_ = start; - return Error(DescribeLastError(), err); - } -yy101: +yy110: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy103: - yych = *++p; - goto yy100; -yy104: +yy112: ++p; -yy105: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy106: +yy114: + yych = *++p; + if (yych == '\n') goto yy134; + goto yy113; +yy115: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy108: +yy117: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy110: +yy119: ++p; yych = *p; - goto yy124; -yy111: + goto yy133; +yy120: { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy112: +yy121: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy114: +yy123: yych = *(q = ++p); if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - goto yy105; -yy115: + goto yy113; +yy124: ++p; yych = *p; if (yybm[0+yych] & 16) { - goto yy115; + goto yy124; } { continue; } -yy118: +yy127: ++p; yych = *p; if (yybm[0+yych] & 32) { - goto yy118; + goto yy127; } - if (yych == '}') goto yy121; + if (yych == '}') goto yy130; p = q; - goto yy105; -yy121: + goto yy113; +yy130: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); continue; } -yy123: +yy132: ++p; yych = *p; -yy124: +yy133: if (yybm[0+yych] & 64) { - goto yy123; + goto yy132; } - goto yy111; -yy125: + goto yy120; +yy134: ++p; yych = *p; -yy126: + if (yych == ' ') goto yy134; + { + continue; + } +yy137: + ++p; + { + if (path) + p = start; + break; + } +yy139: + ++p; + yych = *p; +yy140: if (yybm[0+yych] & 128) { - goto yy125; + goto yy139; } - goto yy96; + goto yy104; } } diff --git a/src/lexer.in.cc b/src/lexer.in.cc index 93d5540..f861239 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -102,8 +102,6 @@ const char* Lexer::TokenErrorHint(Token expected) { string Lexer::DescribeLastError() { if (last_token_) { switch (last_token_[0]) { - case '\r': - return "carriage returns are not allowed, use newlines"; case '\t': return "tabs are not allowed, use spaces"; } @@ -132,8 +130,9 @@ Lexer::Token Lexer::ReadToken() { simple_varname = [a-zA-Z0-9_-]+; varname = [a-zA-Z0-9_.-]+; - [ ]*"#"[^\000\r\n]*"\n" { continue; } - [ ]*[\n] { token = NEWLINE; break; } + [ ]*"#"[^\000\n]*"\n" { continue; } + [ ]*"\r\n" { token = NEWLINE; break; } + [ ]*"\n" { token = NEWLINE; break; } [ ]+ { token = INDENT; break; } "build" { token = BUILD; break; } "pool" { token = POOL; break; } @@ -168,13 +167,15 @@ bool Lexer::PeekToken(Token token) { void Lexer::EatWhitespace() { const char* p = ofs_; + const char* q; for (;;) { ofs_ = p; /*!re2c - [ ]+ { continue; } - "$\n" { continue; } - nul { break; } - [^] { break; } + [ ]+ { continue; } + "$\r\n" { continue; } + "$\n" { continue; } + nul { break; } + [^] { break; } */ } } @@ -207,6 +208,11 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(start, p - start)); continue; } + "\r\n" { + if (path) + p = start; + break; + } [ :|\n] { if (path) { p = start; @@ -226,6 +232,9 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { eval->AddText(StringPiece(" ", 1)); continue; } + "$\r\n"[ ]* { + continue; + } "$\n"[ ]* { continue; } diff --git a/src/lexer_test.cc b/src/lexer_test.cc index e8a1642..331d8e1 100644 --- a/src/lexer_test.cc +++ b/src/lexer_test.cc @@ -14,9 +14,8 @@ #include "lexer.h" -#include <gtest/gtest.h> - #include "eval_env.h" +#include "test.h" TEST(Lexer, ReadVarValue) { Lexer lexer("plain text $var $VaR ${x}\n"); diff --git a/src/line_printer.cc b/src/line_printer.cc index ef1609c..2cd3e17 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -21,6 +21,7 @@ #else #include <unistd.h> #include <sys/ioctl.h> +#include <termios.h> #include <sys/time.h> #endif @@ -49,29 +50,21 @@ void LinePrinter::Print(string to_print, LineType type) { return; } -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO csbi; - GetConsoleScreenBufferInfo(console_, &csbi); -#endif - if (smart_terminal_) { -#ifndef _WIN32 printf("\r"); // Print over previous line, if any. -#else - csbi.dwCursorPosition.X = 0; - SetConsoleCursorPosition(console_, csbi.dwCursorPosition); -#endif + // On Windows, calling a C library function writing to stdout also handles + // pausing the executable when the "Pause" key or Ctrl-S is pressed. } if (smart_terminal_ && type == ELIDE) { #ifdef _WIN32 - // Don't use the full width or console will move to next line. - size_t width = static_cast<size_t>(csbi.dwSize.X) - 1; - to_print = ElideMiddle(to_print, width); - // We don't want to have the cursor spamming back and forth, so - // use WriteConsoleOutput instead which updates the contents of - // the buffer, but doesn't move the cursor position. + CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); + + to_print = ElideMiddle(to_print, static_cast<size_t>(csbi.dwSize.X)); + // We don't want to have the cursor spamming back and forth, so instead of + // printf use WriteConsoleOutput which updates the contents of the buffer, + // but doesn't move the cursor position. COORD buf_size = { csbi.dwSize.X, 1 }; COORD zero_zero = { 0, 0 }; SMALL_RECT target = { @@ -79,16 +72,12 @@ void LinePrinter::Print(string to_print, LineType type) { static_cast<SHORT>(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), csbi.dwCursorPosition.Y }; - CHAR_INFO* char_data = new CHAR_INFO[csbi.dwSize.X]; - memset(char_data, 0, sizeof(CHAR_INFO) * csbi.dwSize.X); - for (int i = 0; i < csbi.dwSize.X; ++i) { - char_data[i].Char.AsciiChar = ' '; + vector<CHAR_INFO> char_data(csbi.dwSize.X); + for (size_t i = 0; i < static_cast<size_t>(csbi.dwSize.X); ++i) { + char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; char_data[i].Attributes = csbi.wAttributes; } - for (size_t i = 0; i < to_print.size(); ++i) - char_data[i].Char.AsciiChar = to_print[i]; - WriteConsoleOutput(console_, char_data, buf_size, zero_zero, &target); - delete[] char_data; + WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); #else // Limit output to width of the terminal if provided so we don't cause // line-wrapping. diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 6fa4f7c..e8c0436 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -24,8 +24,10 @@ #include "util.h" #include "version.h" -ManifestParser::ManifestParser(State* state, FileReader* file_reader) - : state_(state), file_reader_(file_reader) { +ManifestParser::ManifestParser(State* state, FileReader* file_reader, + bool dupe_edge_should_err) + : state_(state), file_reader_(file_reader), + dupe_edge_should_err_(dupe_edge_should_err), quiet_(false) { env_ = &state->bindings_; } @@ -156,7 +158,7 @@ bool ManifestParser::ParseRule(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - if (state_->LookupRule(name) != NULL) + if (env_->LookupRuleCurrentScope(name) != NULL) return lexer_.Error("duplicate rule '" + name + "'", err); Rule* rule = new Rule(name); // XXX scoped_ptr @@ -185,13 +187,13 @@ bool ManifestParser::ParseRule(string* err) { if (rule->bindings_["command"].empty()) return lexer_.Error("expected 'command =' line", err); - state_->AddRule(rule); + env_->AddRule(rule); return true; } bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) { if (!lexer_.ReadIdent(key)) - return false; + return lexer_.Error("expected variable name", err); if (!ExpectToken(Lexer::EQUALS, err)) return false; if (!lexer_.ReadVarValue(value, err)) @@ -209,7 +211,8 @@ bool ManifestParser::ParseDefault(string* err) { do { string path = eval.Evaluate(env_); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; // Unused because this only does lookup. + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); if (!state_->AddDefault(path, &path_err)) return lexer_.Error(path_err, err); @@ -251,7 +254,7 @@ bool ManifestParser::ParseEdge(string* err) { if (!lexer_.ReadIdent(&rule_name)) return lexer_.Error("expected build command name", err); - const Rule* rule = state_->LookupRule(rule_name); + const Rule* rule = env_->LookupRule(rule_name); if (!rule) return lexer_.Error("unknown build rule '" + rule_name + "'", err); @@ -297,16 +300,16 @@ bool ManifestParser::ParseEdge(string* err) { return false; // Bindings on edges are rare, so allocate per-edge envs only when needed. - bool hasIdent = lexer_.PeekToken(Lexer::INDENT); - BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_; - while (hasIdent) { + bool has_indent_token = lexer_.PeekToken(Lexer::INDENT); + BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_; + while (has_indent_token) { string key; EvalString val; if (!ParseLet(&key, &val, err)) return false; env->AddBinding(key, val.Evaluate(env_)); - hasIdent = lexer_.PeekToken(Lexer::INDENT); + has_indent_token = lexer_.PeekToken(Lexer::INDENT); } Edge* edge = state_->AddEdge(rule); @@ -320,19 +323,42 @@ bool ManifestParser::ParseEdge(string* err) { edge->pool_ = pool; } - for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { + edge->outputs_.reserve(outs.size()); + for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) { string path = i->Evaluate(env); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); - state_->AddIn(edge, path); + if (!state_->AddOut(edge, path, slash_bits)) { + if (dupe_edge_should_err_) { + lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]", + err); + return false; + } else if (!quiet_) { + Warning("multiple rules generate %s. " + "builds involving this target will not be correct; " + "continuing anyway [-w dupbuild=warn]", + path.c_str()); + } + } } - for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) { + if (edge->outputs_.empty()) { + // All outputs of the edge are already created by other edges. Don't add + // this edge. Do this check before input nodes are connected to the edge. + state_->edges_.pop_back(); + delete edge; + return true; + } + + edge->inputs_.reserve(ins.size()); + for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(env); string path_err; - if (!CanonicalizePath(&path, &path_err)) + unsigned int slash_bits; + if (!CanonicalizePath(&path, &slash_bits, &path_err)) return lexer_.Error(path_err, err); - state_->AddOut(edge, path); + state_->AddIn(edge, path, slash_bits); } edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; diff --git a/src/manifest_parser.h b/src/manifest_parser.h index 5212f72..f72cd6f 100644 --- a/src/manifest_parser.h +++ b/src/manifest_parser.h @@ -32,13 +32,15 @@ struct ManifestParser { virtual bool ReadFile(const string& path, string* content, string* err) = 0; }; - ManifestParser(State* state, FileReader* file_reader); + ManifestParser(State* state, FileReader* file_reader, + bool dupe_edge_should_err = false); /// Load and parse a file. - bool Load(const string& filename, string* err, Lexer* parent=NULL); + bool Load(const string& filename, string* err, Lexer* parent = NULL); /// Parse a text string of input. Used by tests. bool ParseTest(const string& input, string* err) { + quiet_ = true; return Parse("input", input, err); } @@ -64,6 +66,8 @@ private: BindingEnv* env_; FileReader* file_reader_; Lexer lexer_; + bool dupe_edge_should_err_; + bool quiet_; }; #endif // NINJA_MANIFEST_PARSER_H_ diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc index e40468f..6b56ab0 100644 --- a/src/manifest_parser_perftest.cc +++ b/src/manifest_parser_perftest.cc @@ -16,7 +16,10 @@ // directory. #include <numeric> + +#include <errno.h> #include <stdio.h> +#include <string.h> #ifdef _WIN32 #include "getopt.h" @@ -39,15 +42,19 @@ struct RealFileReader : public ManifestParser::FileReader { } }; -bool WriteFakeManifests(const string& dir) { +bool WriteFakeManifests(const string& dir, string* err) { RealDiskInterface disk_interface; - if (disk_interface.Stat(dir + "/build.ninja") > 0) - return true; + TimeStamp mtime = disk_interface.Stat(dir + "/build.ninja", err); + if (mtime != 0) // 0 means that the file doesn't exist yet. + return mtime != -1; + string command = "python misc/write_fake_manifests.py " + dir; printf("Creating manifest data..."); fflush(stdout); - int err = system(("python misc/write_fake_manifests.py " + dir).c_str()); + int exit_code = system(command.c_str()); printf("done.\n"); - return err == 0; + if (exit_code != 0) + *err = "Failed to run " + command; + return exit_code == 0; } int LoadManifests(bool measure_command_evaluation) { @@ -90,12 +97,14 @@ int main(int argc, char* argv[]) { const char kManifestDir[] = "build/manifest_perftest"; - if (!WriteFakeManifests(kManifestDir)) { - fprintf(stderr, "Failed to write test data\n"); + string err; + if (!WriteFakeManifests(kManifestDir, &err)) { + fprintf(stderr, "Failed to write test data: %s\n", err.c_str()); return 1; } - chdir(kManifestDir); + if (chdir(kManifestDir) < 0) + Fatal("chdir: %s", strerror(errno)); const int kNumRepetitions = 5; vector<int> times; diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 152b965..8f7b575 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -17,18 +17,18 @@ #include <map> #include <vector> -#include <gtest/gtest.h> - #include "graph.h" #include "state.h" +#include "test.h" struct ParserTest : public testing::Test, public ManifestParser::FileReader { void AssertParse(const char* input) { ManifestParser parser(&state, this); string err; - ASSERT_TRUE(parser.ParseTest(input, &err)) << err; + EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); + VerifyGraph(state); } virtual bool ReadFile(const string& path, string* content, string* err) { @@ -61,8 +61,8 @@ TEST_F(ParserTest, Rules) { "\n" "build result: cat in_1.cc in-2.O\n")); - ASSERT_EQ(3u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; + ASSERT_EQ(3u, state.bindings_.GetRules().size()); + const Rule* rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat", rule->name()); EXPECT_EQ("[cat ][$in][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -94,10 +94,10 @@ TEST_F(ParserTest, IgnoreIndentedComments) { "build result: cat in_1.cc in-2.O\n" " #comment\n")); - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; + ASSERT_EQ(2u, state.bindings_.GetRules().size()); + const Rule* rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat", rule->name()); - Edge* edge = state.GetNode("result")->in_edge(); + Edge* edge = state.GetNode("result", 0)->in_edge(); EXPECT_TRUE(edge->GetBindingBool("restat")); EXPECT_FALSE(edge->GetBindingBool("generator")); } @@ -127,8 +127,8 @@ TEST_F(ParserTest, ResponseFiles) { "build out: cat_rsp in\n" " rspfile=out.rsp\n")); - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; + ASSERT_EQ(2u, state.bindings_.GetRules().size()); + const Rule* rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat_rsp", rule->name()); EXPECT_EQ("[cat ][$rspfile][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -144,8 +144,8 @@ TEST_F(ParserTest, InNewline) { "build out: cat_rsp in in2\n" " rspfile=out.rsp\n")); - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; + ASSERT_EQ(2u, state.bindings_.GetRules().size()); + const Rule* rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat_rsp", rule->name()); EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -205,8 +205,8 @@ TEST_F(ParserTest, Continuation) { "build a: link c $\n" " d e f\n")); - ASSERT_EQ(2u, state.rules_.size()); - const Rule* rule = state.rules_.begin()->second; + ASSERT_EQ(2u, state.bindings_.GetRules().size()); + const Rule* rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("link", rule->name()); EXPECT_EQ("[foo bar baz]", rule->GetBinding("command")->Serialize()); } @@ -270,6 +270,26 @@ TEST_F(ParserTest, CanonicalizeFile) { EXPECT_FALSE(state.LookupNode("in//2")); } +#ifdef _WIN32 +TEST_F(ParserTest, CanonicalizeFileBackslashes) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build out: cat in\\1 in\\\\2\n" +"build in\\1: cat\n" +"build in\\2: cat\n")); + + Node* node = state.LookupNode("in/1");; + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); + node = state.LookupNode("in/2"); + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); + EXPECT_FALSE(state.LookupNode("in//1")); + EXPECT_FALSE(state.LookupNode("in//2")); +} +#endif + TEST_F(ParserTest, PathVariables) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" @@ -293,6 +313,70 @@ TEST_F(ParserTest, CanonicalizePaths) { EXPECT_TRUE(state.LookupNode("bar/foo.cc")); } +#ifdef _WIN32 +TEST_F(ParserTest, CanonicalizePathsBackslashes) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build ./out.o: cat ./bar/baz/../foo.cc\n" +"build .\\out2.o: cat .\\bar/baz\\..\\foo.cc\n" +"build .\\out3.o: cat .\\bar\\baz\\..\\foo3.cc\n" +)); + + EXPECT_FALSE(state.LookupNode("./out.o")); + EXPECT_FALSE(state.LookupNode(".\\out2.o")); + EXPECT_FALSE(state.LookupNode(".\\out3.o")); + EXPECT_TRUE(state.LookupNode("out.o")); + EXPECT_TRUE(state.LookupNode("out2.o")); + EXPECT_TRUE(state.LookupNode("out3.o")); + EXPECT_FALSE(state.LookupNode("./bar/baz/../foo.cc")); + EXPECT_FALSE(state.LookupNode(".\\bar/baz\\..\\foo.cc")); + EXPECT_FALSE(state.LookupNode(".\\bar/baz\\..\\foo3.cc")); + Node* node = state.LookupNode("bar/foo.cc"); + EXPECT_TRUE(node); + EXPECT_EQ(0, node->slash_bits()); + node = state.LookupNode("bar/foo3.cc"); + EXPECT_TRUE(node); + EXPECT_EQ(1, node->slash_bits()); +} +#endif + +TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputs) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build out1 out2: cat in1\n" +"build out1: cat in2\n" +"build final: cat out1\n" +)); + // AssertParse() checks that the generated build graph is self-consistent. + // That's all the checking that this test needs. +} + +TEST_F(ParserTest, NoDeadPointerFromDuplicateEdge) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n" +" command = cat $in > $out\n" +"build out: cat in\n" +"build out: cat in\n" +)); + // AssertParse() checks that the generated build graph is self-consistent. + // That's all the checking that this test needs. +} + +TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputsError) { + const char kInput[] = +"rule cat\n" +" command = cat $in > $out\n" +"build out1 out2: cat in1\n" +"build out1: cat in2\n" +"build final: cat out1\n"; + ManifestParser parser(&state, this, /*dupe_edges_should_err=*/true); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:5: multiple rules generate out1 [-w dupbuild=err]\n", err); +} + TEST_F(ParserTest, ReservedWords) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule build\n" @@ -553,6 +637,15 @@ TEST_F(ParserTest, Errors) { State state; ManifestParser parser(&state, NULL); string err; + EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n && bar", + &err)); + EXPECT_EQ("input:3: expected variable name\n", err); + } + + { + State state; + ManifestParser parser(&state, NULL); + string err; EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n" "build $: cc bar.cc\n", &err)); @@ -767,18 +860,27 @@ TEST_F(ParserTest, MissingSubNinja) { } TEST_F(ParserTest, DuplicateRuleInDifferentSubninjas) { - // Test that rules live in a global namespace and aren't scoped to subninjas. + // Test that rules are scoped to subninjas. files_["test.ninja"] = "rule cat\n" " command = cat\n"; ManifestParser parser(&state, this); string err; - EXPECT_FALSE(parser.ParseTest("rule cat\n" + EXPECT_TRUE(parser.ParseTest("rule cat\n" " command = cat\n" "subninja test.ninja\n", &err)); - EXPECT_EQ("test.ninja:1: duplicate rule 'cat'\n" - "rule cat\n" - " ^ near here" - , err); +} + +TEST_F(ParserTest, DuplicateRuleInDifferentSubninjasWithInclude) { + // Test that rules are scoped to subninjas even with includes. + files_["rules.ninja"] = "rule cat\n" + " command = cat\n"; + files_["test.ninja"] = "include rules.ninja\n" + "build x : cat\n"; + ManifestParser parser(&state, this); + string err; + EXPECT_TRUE(parser.ParseTest("include rules.ninja\n" + "subninja test.ninja\n" + "build y : cat\n", &err)); } TEST_F(ParserTest, Include) { @@ -835,6 +937,16 @@ TEST_F(ParserTest, DefaultDefault) { EXPECT_EQ("", err); } +TEST_F(ParserTest, DefaultDefaultCycle) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build a: cat a\n")); + + string err; + EXPECT_EQ(0u, state.DefaultNodes(&err).size()); + EXPECT_EQ("could not determine root nodes of build graph", err); +} + TEST_F(ParserTest, DefaultStatements) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n command = cat $in > $out\n" @@ -862,22 +974,18 @@ TEST_F(ParserTest, UTF8) { " description = compilaci\xC3\xB3\n")); } -// We might want to eventually allow CRLF to be nice to Windows developers, -// but for now just verify we error out with a nice message. TEST_F(ParserTest, CRLF) { State state; ManifestParser parser(&state, NULL); string err; - EXPECT_FALSE(parser.ParseTest("# comment with crlf\r\n", - &err)); - EXPECT_EQ("input:1: lexing error\n", - err); - - EXPECT_FALSE(parser.ParseTest("foo = foo\nbar = bar\r\n", - &err)); - EXPECT_EQ("input:2: carriage returns are not allowed, use newlines\n" - "bar = bar\r\n" - " ^ near here", - err); + EXPECT_TRUE(parser.ParseTest("# comment with crlf\r\n", &err)); + EXPECT_TRUE(parser.ParseTest("foo = foo\nbar = bar\r\n", &err)); + EXPECT_TRUE(parser.ParseTest( + "pool link_pool\r\n" + " depth = 15\r\n\r\n" + "rule xyz\r\n" + " command = something$expand \r\n" + " description = YAY!\r\n", + &err)); } diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index c79ec0e..c611919 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef NINJA_BOOTSTRAP +#ifdef _MSC_VER #include <windows.h> #include <DbgHelp.h> - #include "util.h" typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( @@ -85,4 +84,4 @@ void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { Warning("minidump created: %s", temp_file); } -#endif // NINJA_BOOTSTRAP +#endif // _MSC_VER diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index e465279..d516240 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -15,6 +15,7 @@ #include "msvc_helper.h" #include <algorithm> +#include <assert.h> #include <stdio.h> #include <string.h> #include <windows.h> @@ -82,10 +83,10 @@ bool CLParser::FilterInputFilename(string line) { EndsWith(line, ".cpp"); } -string CLParser::Parse(const string& output, const string& deps_prefix) { - string filtered_output; - +bool CLParser::Parse(const string& output, const string& deps_prefix, + string* filtered_output, string* err) { // Loop over all lines in the output to process them. + assert(&output != filtered_output); size_t start = 0; while (start < output.size()) { size_t end = output.find_first_of("\r\n", start); @@ -95,16 +96,18 @@ string CLParser::Parse(const string& output, const string& deps_prefix) { string include = FilterShowIncludes(line, deps_prefix); if (!include.empty()) { - include = IncludesNormalize::Normalize(include, NULL); - if (!IsSystemInclude(include)) - includes_.insert(include); + string normalized; + if (!IncludesNormalize::Normalize(include, NULL, &normalized, err)) + return false; + if (!IsSystemInclude(normalized)) + includes_.insert(normalized); } else if (FilterInputFilename(line)) { // Drop it. // TODO: if we support compiling multiple output files in a single // cl.exe invocation, we should stash the filename. } else { - filtered_output.append(line); - filtered_output.append("\n"); + filtered_output->append(line); + filtered_output->append("\n"); } if (end < output.size() && output[end] == '\r') @@ -114,7 +117,7 @@ string CLParser::Parse(const string& output, const string& deps_prefix) { start = end; } - return filtered_output; + return true; } int CLWrapper::Run(const string& command, string* output) { diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 5d7dcb0..30f87f3 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -40,9 +40,11 @@ struct CLParser { /// Exposed for testing. static bool FilterInputFilename(string line); - /// Parse the full output of cl, returning the output (if any) that - /// should printed. - string Parse(const string& output, const string& deps_prefix); + /// Parse the full output of cl, filling filtered_output with the text that + /// should be printed (if any). Returns true on success, or false with err + /// filled. output must not be the same object as filtered_object. + bool Parse(const string& output, const string& deps_prefix, + string* filtered_output, string* err); set<string> includes_; }; diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 58bc797..680aaad 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -127,7 +127,9 @@ int MSVCHelperMain(int argc, char** argv) { if (output_filename) { CLParser parser; - output = parser.Parse(output, deps_prefix); + string err; + if (!parser.Parse(output, deps_prefix, &output, &err)) + Fatal("%s\n", err.c_str()); WriteDepFileOrDie(output_filename, parser); } diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 391c045..49d27c1 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -14,8 +14,6 @@ #include "msvc_helper.h" -#include <gtest/gtest.h> - #include "test.h" #include "util.h" @@ -48,11 +46,12 @@ TEST(CLParserTest, FilterInputFilename) { TEST(CLParserTest, ParseSimple) { CLParser parser; - string output = parser.Parse( + string output, err; + ASSERT_TRUE(parser.Parse( "foo\r\n" "Note: inc file prefix: foo.h\r\n" "bar\r\n", - "Note: inc file prefix:"); + "Note: inc file prefix:", &output, &err)); ASSERT_EQ("foo\nbar\n", output); ASSERT_EQ(1u, parser.includes_.size()); @@ -61,20 +60,22 @@ TEST(CLParserTest, ParseSimple) { TEST(CLParserTest, ParseFilenameFilter) { CLParser parser; - string output = parser.Parse( + string output, err; + ASSERT_TRUE(parser.Parse( "foo.cc\r\n" "cl: warning\r\n", - ""); + "", &output, &err)); ASSERT_EQ("cl: warning\n", output); } TEST(CLParserTest, ParseSystemInclude) { CLParser parser; - string output = parser.Parse( + string output, err; + ASSERT_TRUE(parser.Parse( "Note: including file: c:\\Program Files\\foo.h\r\n" "Note: including file: d:\\Microsoft Visual Studio\\bar.h\r\n" "Note: including file: path.h\r\n", - ""); + "", &output, &err)); // We should have dropped the first two includes because they look like // system headers. ASSERT_EQ("", output); @@ -84,11 +85,12 @@ TEST(CLParserTest, ParseSystemInclude) { TEST(CLParserTest, DuplicatedHeader) { CLParser parser; - string output = parser.Parse( + string output, err; + ASSERT_TRUE(parser.Parse( "Note: including file: foo.h\r\n" "Note: including file: bar.h\r\n" "Note: including file: foo.h\r\n", - ""); + "", &output, &err)); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); ASSERT_EQ(2u, parser.includes_.size()); @@ -96,11 +98,12 @@ TEST(CLParserTest, DuplicatedHeader) { TEST(CLParserTest, DuplicatedHeaderPathConverted) { CLParser parser; - string output = parser.Parse( + string output, err; + ASSERT_TRUE(parser.Parse( "Note: including file: sub/foo.h\r\n" "Note: including file: bar.h\r\n" "Note: including file: sub\\foo.h\r\n", - ""); + "", &output, &err)); // We should have dropped one copy of foo.h. ASSERT_EQ("", output); ASSERT_EQ(2u, parser.includes_.size()); diff --git a/src/ninja.cc b/src/ninja.cc index 50de43e..fe4a580 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -64,6 +64,9 @@ struct Options { /// Tool to run rather than building. const Tool* tool; + + /// Whether duplicate rules for one target should warn or print an error. + bool dupe_edges_should_err; }; /// The Ninja main() loads up a series of data structures; various tools need @@ -140,6 +143,8 @@ struct NinjaMain : public BuildLogUser { virtual bool IsPathDead(StringPiece s) const { Node* n = state_.LookupNode(s); + if (!n || !n->in_edge()) + return false; // Just checking n isn't enough: If an old output is both in the build log // and in the deps log, it will have a Node object in state_. (It will also // have an in edge if one of its inputs is another output that's in the deps @@ -147,7 +152,13 @@ struct NinjaMain : public BuildLogUser { // edge is rare, and the first recompaction will delete all old outputs from // the deps log, and then a second recompaction will clear the build log, // which seems good enough for this corner case.) - return !n || !n->in_edge(); + // Do keep entries around for files which still exist on disk, for + // generators that want to use this information. + string err; + TimeStamp mtime = disk_interface_.Stat(s.AsString(), &err); + if (mtime == -1) + Error("%s", err.c_str()); // Log and ignore Stat() errors. + return mtime == 0; } }; @@ -161,7 +172,8 @@ struct Tool { /// When to run the tool. enum { - /// Run after parsing the command-line flags (as early as possible). + /// Run after parsing the command-line flags and potentially changing + /// the current working directory (as early as possible). RUN_AFTER_FLAGS, /// Run after loading build.ninja. @@ -189,17 +201,15 @@ void Usage(const BuildConfig& config) { " -f FILE specify input build file [default=build.ninja]\n" "\n" " -j N run N jobs in parallel [default=%d, derived from CPUs available]\n" -" -l N do not start new jobs if the load average is greater than N\n" -#ifdef _WIN32 -" (not yet implemented on Windows)\n" -#endif " -k N keep going until N jobs fail [default=1]\n" +" -l N do not start new jobs if the load average is greater than N\n" " -n dry run (don't run commands but act like they succeeded)\n" " -v show all command lines while building\n" "\n" " -d MODE enable debugging (use -d list to list modes)\n" " -t TOOL run a subtool (use -t list to list subtools)\n" -" terminates toplevel options; further flags are passed to the tool\n", +" terminates toplevel options; further flags are passed to the tool\n" +" -w FLAG adjust warnings (use -w list to list warnings)\n", kNinjaVersion, config.parallelism); } @@ -228,7 +238,8 @@ struct RealFileReader : public ManifestParser::FileReader { /// Returns true if the manifest was rebuilt. bool NinjaMain::RebuildManifest(const char* input_file, string* err) { string path = input_file; - if (!CanonicalizePath(&path, err)) + unsigned int slash_bits; // Unused because this path is only used for lookup. + if (!CanonicalizePath(&path, &slash_bits, err)) return false; Node* node = state_.LookupNode(path); if (!node) @@ -240,17 +251,17 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err) { if (builder.AlreadyUpToDate()) return false; // Not an error, but we didn't rebuild. - if (!builder.Build(err)) - return false; - // The manifest was only rebuilt if it is now dirty (it may have been cleaned - // by a restat). - return node->dirty(); + // Even if the manifest was cleaned by a restat rule, claim that it was + // rebuilt. Not doing so can lead to crashes, see + // https://github.com/ninja-build/ninja/issues/874 + return builder.Build(err); } Node* NinjaMain::CollectTarget(const char* cpath, string* err) { string path = cpath; - if (!CanonicalizePath(&path, err)) + unsigned int slash_bits; + if (!CanonicalizePath(&path, &slash_bits, err)) return NULL; // Special syntax: "foo.cc^" means "the first output of foo.cc". @@ -276,8 +287,8 @@ Node* NinjaMain::CollectTarget(const char* cpath, string* err) { } return node; } else { - *err = "unknown target '" + path + "'"; - + *err = + "unknown target '" + Node::PathDecanonicalized(path, slash_bits) + "'"; if (path == "clean") { *err += ", did you mean 'ninja -t clean'?"; } else if (path == "help") { @@ -363,7 +374,7 @@ int NinjaMain::ToolQuery(int argc, char* argv[]) { return 0; } -#if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) +#if defined(NINJA_HAVE_BROWSE) int NinjaMain::ToolBrowse(int argc, char* argv[]) { if (argc < 1) { Error("expected a target to browse"); @@ -476,7 +487,10 @@ int NinjaMain::ToolDeps(int argc, char** argv) { continue; } - TimeStamp mtime = disk_interface.Stat((*it)->path()); + string err; + TimeStamp mtime = disk_interface.Stat((*it)->path(), &err); + if (mtime == -1) + Error("%s", err.c_str()); // Log and ignore Stat() errors; printf("%s: #deps %d, deps mtime %d (%s)\n", (*it)->path().c_str(), deps->node_count, deps->mtime, (!mtime || mtime > deps->mtime ? "STALE":"VALID")); @@ -696,7 +710,7 @@ int NinjaMain::ToolUrtle(int argc, char** argv) { /// Returns a Tool, or NULL if Ninja should exit. const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { -#if !defined(_WIN32) && !defined(NINJA_BOOTSTRAP) +#if defined(NINJA_HAVE_BROWSE) { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, #endif @@ -760,6 +774,9 @@ bool DebugEnable(const string& name) { " stats print operation counts/timing info\n" " explain explain what caused a command to execute\n" " keeprsp don't delete @response files on success\n" +#ifdef _WIN32 +" nostatcache don't batch stat() calls per directory and cache them\n" +#endif "multiple modes can be enabled via -d FOO -d BAR\n"); return false; } else if (name == "stats") { @@ -771,9 +788,13 @@ bool DebugEnable(const string& name) { } else if (name == "keeprsp") { g_keep_rsp = true; return true; + } else if (name == "nostatcache") { + g_experimental_statcache = false; + return true; } else { const char* suggestion = - SpellcheckString(name.c_str(), "stats", "explain", NULL); + SpellcheckString(name.c_str(), "stats", "explain", "keeprsp", + "nostatcache", NULL); if (suggestion) { Error("unknown debug setting '%s', did you mean '%s'?", name.c_str(), suggestion); @@ -784,6 +805,32 @@ bool DebugEnable(const string& name) { } } +/// Set a warning flag. Returns false if Ninja should exit instead of +/// continuing. +bool WarningEnable(const string& name, Options* options) { + if (name == "list") { + printf("warning flags:\n" +" dupbuild={err,warn} multiple build lines for one target\n"); + return false; + } else if (name == "dupbuild=err") { + options->dupe_edges_should_err = true; + return true; + } else if (name == "dupbuild=warn") { + options->dupe_edges_should_err = false; + return true; + } else { + const char* suggestion = + SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn", NULL); + if (suggestion) { + Error("unknown warning flag '%s', did you mean '%s'?", + name.c_str(), suggestion); + } else { + Error("unknown warning flag '%s'", name.c_str()); + } + return false; + } +} + bool NinjaMain::OpenBuildLog(bool recompact_only) { string log_path = ".ninja_log"; if (!build_dir_.empty()) @@ -882,6 +929,8 @@ int NinjaMain::RunBuild(int argc, char** argv) { return 1; } + disk_interface_.AllowStatCache(g_experimental_statcache); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { @@ -895,6 +944,9 @@ int NinjaMain::RunBuild(int argc, char** argv) { } } + // Make sure restat rules do not see stale timestamps. + disk_interface_.AllowStatCache(false); + if (builder.AlreadyUpToDate()) { printf("ninja: no work to do.\n"); return 0; @@ -949,7 +1001,7 @@ int ReadFlags(int* argc, char*** argv, int opt; while (!options->tool && - (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vC:h", kLongOptions, + (opt = getopt_long(*argc, *argv, "d:f:j:k:l:nt:vw:C:h", kLongOptions, NULL)) != -1) { switch (opt) { case 'd': @@ -998,6 +1050,10 @@ int ReadFlags(int* argc, char*** argv, case 'v': config->verbosity = BuildConfig::VERBOSE; break; + case 'w': + if (!WarningEnable(optarg, options)) + return 1; + break; case 'C': options->working_dir = optarg; break; @@ -1028,13 +1084,6 @@ int real_main(int argc, char** argv) { if (exit_code >= 0) return exit_code; - if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { - // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed - // by other tools. - NinjaMain ninja(ninja_command, config); - return (ninja.*options.tool->func)(argc, argv); - } - if (options.working_dir) { // The formatting of this string, complete with funny quotes, is // so Emacs can properly identify that the cwd has changed for @@ -1048,13 +1097,21 @@ int real_main(int argc, char** argv) { } } - // The build can take up to 2 passes: one to rebuild the manifest, then - // another to build the desired target. - for (int cycle = 0; cycle < 2; ++cycle) { + if (options.tool && options.tool->when == Tool::RUN_AFTER_FLAGS) { + // None of the RUN_AFTER_FLAGS actually use a NinjaMain, but it's needed + // by other tools. + NinjaMain ninja(ninja_command, config); + return (ninja.*options.tool->func)(argc, argv); + } + + // Limit number of rebuilds, to prevent infinite loops. + const int kCycleLimit = 100; + for (int cycle = 1; cycle <= kCycleLimit; ++cycle) { NinjaMain ninja(ninja_command, config); RealFileReader file_reader; - ManifestParser parser(&ninja.state_, &file_reader); + ManifestParser parser(&ninja.state_, &file_reader, + options.dupe_edges_should_err); string err; if (!parser.Load(options.input_file, &err)) { Error("%s", err.c_str()); @@ -1073,16 +1130,13 @@ int real_main(int argc, char** argv) { if (options.tool && options.tool->when == Tool::RUN_AFTER_LOGS) return (ninja.*options.tool->func)(argc, argv); - // The first time through, attempt to rebuild the manifest before - // building anything else. - if (cycle == 0) { - if (ninja.RebuildManifest(options.input_file, &err)) { - // Start the build over with the new manifest. - continue; - } else if (!err.empty()) { - Error("rebuilding '%s': %s", options.input_file, err.c_str()); - return 1; - } + // Attempt to rebuild the manifest before building anything else + if (ninja.RebuildManifest(options.input_file, &err)) { + // Start the build over with the new manifest. + continue; + } else if (!err.empty()) { + Error("rebuilding '%s': %s", options.input_file, err.c_str()); + return 1; } int result = ninja.RunBuild(argc, argv); @@ -1091,16 +1145,18 @@ int real_main(int argc, char** argv) { return result; } - return 1; // Shouldn't be reached. + Error("manifest '%s' still dirty after %d tries\n", + options.input_file, kCycleLimit); + return 1; } } // anonymous namespace int main(int argc, char** argv) { -#if !defined(NINJA_BOOTSTRAP) && defined(_MSC_VER) +#if defined(_MSC_VER) // Set a handler to catch crashes not caught by the __try..__except // block (e.g. an exception in a stack-unwind-block). - set_terminate(TerminateHandler); + std::set_terminate(TerminateHandler); __try { // Running inside __try ... __except suppresses any Windows error // dialogs for errors such as bad_alloc. diff --git a/src/ninja_test.cc b/src/ninja_test.cc index 989ea5c..54d8784 100644 --- a/src/ninja_test.cc +++ b/src/ninja_test.cc @@ -15,9 +15,35 @@ #include <stdarg.h> #include <stdio.h> -#include "gtest/gtest.h" +#ifdef _WIN32 +#include "getopt.h" +#else +#include <getopt.h> +#endif + +#include "test.h" #include "line_printer.h" +struct RegisteredTest { + testing::Test* (*factory)(); + const char *name; + bool should_run; +}; +// This can't be a vector because tests call RegisterTest from static +// initializers and the order static initializers run it isn't specified. So +// the vector constructor isn't guaranteed to run before all of the +// RegisterTest() calls. +static RegisteredTest tests[10000]; +testing::Test* g_current_test; +static int ntests; +static LinePrinter printer; + +void RegisterTest(testing::Test* (*factory)(), const char* name) { + tests[ntests].factory = factory; + tests[ntests++].name = name; +} + +namespace { string StringPrintf(const char* format, ...) { const int N = 1024; char buf[N]; @@ -30,59 +56,101 @@ string StringPrintf(const char* format, ...) { return buf; } -/// A test result printer that's less wordy than gtest's default. -struct LaconicPrinter : public testing::EmptyTestEventListener { - LaconicPrinter() : tests_started_(0), test_count_(0), iteration_(0) {} - virtual void OnTestProgramStart(const testing::UnitTest& unit_test) { - test_count_ = unit_test.test_to_run_count(); - } +void Usage() { + fprintf(stderr, +"usage: ninja_tests [options]\n" +"\n" +"options:\n" +" --gtest_filter=POSTIVE_PATTERN[-NEGATIVE_PATTERN]\n" +" Run tests whose names match the positive but not the negative pattern.\n" +" '*' matches any substring. (gtest's ':', '?' are not implemented).\n"); +} - virtual void OnTestIterationStart(const testing::UnitTest& test_info, - int iteration) { - tests_started_ = 0; - iteration_ = iteration; +bool PatternMatchesString(const char* pattern, const char* str) { + switch (*pattern) { + case '\0': + case '-': return *str == '\0'; + case '*': return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); } +} - virtual void OnTestStart(const testing::TestInfo& test_info) { - ++tests_started_; - printer_.Print( - StringPrintf("[%d/%d%s] %s.%s", - tests_started_, - test_count_, - iteration_ ? StringPrintf(" iter %d", iteration_).c_str() - : "", - test_info.test_case_name(), - test_info.name()), - LinePrinter::ELIDE); - } +bool TestMatchesFilter(const char* test, const char* filter) { + // Split --gtest_filter at '-' into positive and negative filters. + const char* const dash = strchr(filter, '-'); + const char* pos = dash == filter ? "*" : filter; //Treat '-test1' as '*-test1' + const char* neg = dash ? dash + 1 : ""; + return PatternMatchesString(pos, test) && !PatternMatchesString(neg, test); +} - virtual void OnTestPartResult( - const testing::TestPartResult& test_part_result) { - if (!test_part_result.failed()) - return; - printer_.PrintOnNewLine(StringPrintf( - "*** Failure in %s:%d\n%s\n", test_part_result.file_name(), - test_part_result.line_number(), test_part_result.summary())); - } +bool ReadFlags(int* argc, char*** argv, const char** test_filter) { + enum { OPT_GTEST_FILTER = 1 }; + const option kLongOptions[] = { + { "gtest_filter", required_argument, NULL, OPT_GTEST_FILTER }, + { NULL, 0, NULL, 0 } + }; - virtual void OnTestProgramEnd(const testing::UnitTest& unit_test) { - printer_.PrintOnNewLine(unit_test.Passed() ? "passed\n" : "failed\n"); + int opt; + while ((opt = getopt_long(*argc, *argv, "h", kLongOptions, NULL)) != -1) { + switch (opt) { + case OPT_GTEST_FILTER: + if (strchr(optarg, '?') == NULL && strchr(optarg, ':') == NULL) { + *test_filter = optarg; + break; + } // else fall through. + default: + Usage(); + return false; + } } + *argv += optind; + *argc -= optind; + return true; +} - private: - LinePrinter printer_; - int tests_started_; - int test_count_; - int iteration_; -}; +} // namespace + +bool testing::Test::Check(bool condition, const char* file, int line, + const char* error) { + if (!condition) { + printer.PrintOnNewLine( + StringPrintf("*** Failure in %s:%d\n%s\n", file, line, error)); + failed_ = true; + } + return condition; +} int main(int argc, char **argv) { - testing::InitGoogleTest(&argc, argv); + int tests_started = 0; - testing::TestEventListeners& listeners = - testing::UnitTest::GetInstance()->listeners(); - delete listeners.Release(listeners.default_result_printer()); - listeners.Append(new LaconicPrinter); + const char* test_filter = "*"; + if (!ReadFlags(&argc, &argv, &test_filter)) + return 1; + + int nactivetests = 0; + for (int i = 0; i < ntests; i++) + if ((tests[i].should_run = TestMatchesFilter(tests[i].name, test_filter))) + ++nactivetests; + + bool passed = true; + for (int i = 0; i < ntests; i++) { + if (!tests[i].should_run) continue; + + ++tests_started; + testing::Test* test = tests[i].factory(); + printer.Print( + StringPrintf("[%d/%d] %s", tests_started, nactivetests, tests[i].name), + LinePrinter::ELIDE); + test->SetUp(); + test->Run(); + test->TearDown(); + if (test->Failed()) + passed = false; + delete test; + } - return RUN_ALL_TESTS(); + printer.PrintOnNewLine(passed ? "passed\n" : "failed\n"); + return passed ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/src/state.cc b/src/state.cc index 7258272..a70f211 100644 --- a/src/state.cc +++ b/src/state.cc @@ -61,6 +61,7 @@ void Pool::Dump() const { } } +// static bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { if (!a) return b; if (!b) return false; @@ -73,23 +74,11 @@ Pool State::kConsolePool("console", 1); const Rule State::kPhonyRule("phony"); State::State() { - AddRule(&kPhonyRule); + bindings_.AddRule(&kPhonyRule); AddPool(&kDefaultPool); AddPool(&kConsolePool); } -void State::AddRule(const Rule* rule) { - assert(LookupRule(rule->name()) == NULL); - rules_[rule->name()] = rule; -} - -const Rule* State::LookupRule(const string& rule_name) { - map<string, const Rule*>::iterator i = rules_.find(rule_name); - if (i == rules_.end()) - return NULL; - return i->second; -} - void State::AddPool(Pool* pool) { assert(LookupPool(pool->name()) == NULL); pools_[pool->name()] = pool; @@ -111,11 +100,11 @@ Edge* State::AddEdge(const Rule* rule) { return edge; } -Node* State::GetNode(StringPiece path) { +Node* State::GetNode(StringPiece path, unsigned int slash_bits) { Node* node = LookupNode(path); if (node) return node; - node = new Node(path.AsString()); + node = new Node(path.AsString(), slash_bits); paths_[node->path()] = node; return node; } @@ -145,22 +134,19 @@ Node* State::SpellcheckNode(const string& path) { return result; } -void State::AddIn(Edge* edge, StringPiece path) { - Node* node = GetNode(path); +void State::AddIn(Edge* edge, StringPiece path, unsigned int slash_bits) { + Node* node = GetNode(path, slash_bits); edge->inputs_.push_back(node); node->AddOutEdge(edge); } -void State::AddOut(Edge* edge, StringPiece path) { - Node* node = GetNode(path); +bool State::AddOut(Edge* edge, StringPiece path, unsigned int slash_bits) { + Node* node = GetNode(path, slash_bits); + if (node->in_edge()) + return false; edge->outputs_.push_back(node); - if (node->in_edge()) { - Warning("multiple rules generate %s. " - "builds involving this target will not be correct; " - "continuing anyway", - path.AsString().c_str()); - } node->set_in_edge(edge); + return true; } bool State::AddDefault(StringPiece path, string* err) { @@ -187,7 +173,6 @@ vector<Node*> State::RootNodes(string* err) { if (!edges_.empty() && root_nodes.empty()) *err = "could not determine root nodes of build graph"; - assert(edges_.empty() || !root_nodes.empty()); return root_nodes; } diff --git a/src/state.h b/src/state.h index c382dc0..d7987ba 100644 --- a/src/state.h +++ b/src/state.h @@ -37,13 +37,14 @@ struct Rule; /// the total scheduled weight diminishes enough (i.e. when a scheduled edge /// completes). struct Pool { - explicit Pool(const string& name, int depth) - : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) { } + Pool(const string& name, int depth) + : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) {} // A depth of 0 is infinite bool is_valid() const { return depth_ >= 0; } int depth() const { return depth_; } const string& name() const { return name_; } + int current_use() const { return current_use_; } /// true if the Pool might delay this edge bool ShouldDelayEdge() const { return depth_ != 0; } @@ -79,7 +80,7 @@ struct Pool { DelayedEdges delayed_; }; -/// Global state (file status, loaded rules) for a single run. +/// Global state (file status) for a single run. struct State { static Pool kDefaultPool; static Pool kConsolePool; @@ -87,20 +88,17 @@ struct State { State(); - void AddRule(const Rule* rule); - const Rule* LookupRule(const string& rule_name); - void AddPool(Pool* pool); Pool* LookupPool(const string& pool_name); Edge* AddEdge(const Rule* rule); - Node* GetNode(StringPiece path); + Node* GetNode(StringPiece path, unsigned int slash_bits); Node* LookupNode(StringPiece path) const; Node* SpellcheckNode(const string& path); - void AddIn(Edge* edge, StringPiece path); - void AddOut(Edge* edge, StringPiece path); + void AddIn(Edge* edge, StringPiece path, unsigned int slash_bits); + bool AddOut(Edge* edge, StringPiece path, unsigned int slash_bits); bool AddDefault(StringPiece path, string* error); /// Reset state. Keeps all nodes and edges, but restores them to the @@ -119,9 +117,6 @@ struct State { typedef ExternalStringHashMap<Node*>::Type Paths; Paths paths_; - /// All the rules used in the graph. - map<string, const Rule*> rules_; - /// All the pools used in the graph. map<string, Pool*> pools_; diff --git a/src/state_test.cc b/src/state_test.cc index af2bff1..458b519 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <gtest/gtest.h> - #include "graph.h" #include "state.h" +#include "test.h" namespace { @@ -30,18 +29,18 @@ TEST(State, Basic) { Rule* rule = new Rule("cat"); rule->AddBinding("command", command); - state.AddRule(rule); + state.bindings_.AddRule(rule); Edge* edge = state.AddEdge(rule); - state.AddIn(edge, "in1"); - state.AddIn(edge, "in2"); - state.AddOut(edge, "out"); + state.AddIn(edge, "in1", 0); + state.AddIn(edge, "in2", 0); + state.AddOut(edge, "out", 0); EXPECT_EQ("cat in1 in2 > out", edge->EvaluateCommand()); - EXPECT_FALSE(state.GetNode("in1")->dirty()); - EXPECT_FALSE(state.GetNode("in2")->dirty()); - EXPECT_FALSE(state.GetNode("out")->dirty()); + EXPECT_FALSE(state.GetNode("in1", 0)->dirty()); + EXPECT_FALSE(state.GetNode("in2", 0)->dirty()); + EXPECT_FALSE(state.GetNode("out", 0)->dirty()); } } // namespace diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index 743e406..f3baec2 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -28,6 +28,7 @@ Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1), use_console_(use_console) { } + Subprocess::~Subprocess() { if (fd_ >= 0) close(fd_); @@ -59,14 +60,19 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // Track which fd we use to report errors on. int error_pipe = output_pipe[1]; do { - if (sigaction(SIGINT, &set->old_act_, 0) < 0) + if (sigaction(SIGINT, &set->old_int_act_, 0) < 0) + break; + if (sigaction(SIGTERM, &set->old_term_act_, 0) < 0) break; if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) break; if (!use_console_) { - // Put the child in its own process group, so ctrl-c won't reach it. - if (setpgid(0, 0) < 0) + // Put the child in its own session and process group. It will be + // detached from the current terminal and ctrl-c won't reach it. + // Since this process was just forked, it is not a process group leader + // and setsid() will succeed. + if (setsid() < 0) break; // Open /dev/null over stdin. @@ -130,7 +136,7 @@ ExitStatus Subprocess::Finish() { if (exit == 0) return ExitSuccess; } else if (WIFSIGNALED(status)) { - if (WTERMSIG(status) == SIGINT) + if (WTERMSIG(status) == SIGINT || WTERMSIG(status) == SIGTERM) return ExitInterrupted; } return ExitFailure; @@ -144,31 +150,48 @@ const string& Subprocess::GetOutput() const { return buf_; } -bool SubprocessSet::interrupted_; +int SubprocessSet::interrupted_; void SubprocessSet::SetInterruptedFlag(int signum) { - (void) signum; - interrupted_ = true; + interrupted_ = signum; +} + +void SubprocessSet::HandlePendingInterruption() { + sigset_t pending; + sigemptyset(&pending); + if (sigpending(&pending) == -1) { + perror("ninja: sigpending"); + return; + } + if (sigismember(&pending, SIGINT)) + interrupted_ = SIGINT; + else if (sigismember(&pending, SIGTERM)) + interrupted_ = SIGTERM; } SubprocessSet::SubprocessSet() { sigset_t set; sigemptyset(&set); sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); if (sigprocmask(SIG_BLOCK, &set, &old_mask_) < 0) Fatal("sigprocmask: %s", strerror(errno)); struct sigaction act; memset(&act, 0, sizeof(act)); act.sa_handler = SetInterruptedFlag; - if (sigaction(SIGINT, &act, &old_act_) < 0) + if (sigaction(SIGINT, &act, &old_int_act_) < 0) + Fatal("sigaction: %s", strerror(errno)); + if (sigaction(SIGTERM, &act, &old_term_act_) < 0) Fatal("sigaction: %s", strerror(errno)); } SubprocessSet::~SubprocessSet() { Clear(); - if (sigaction(SIGINT, &old_act_, 0) < 0) + if (sigaction(SIGINT, &old_int_act_, 0) < 0) + Fatal("sigaction: %s", strerror(errno)); + if (sigaction(SIGTERM, &old_term_act_, 0) < 0) Fatal("sigaction: %s", strerror(errno)); if (sigprocmask(SIG_SETMASK, &old_mask_, 0) < 0) Fatal("sigprocmask: %s", strerror(errno)); @@ -199,16 +222,20 @@ bool SubprocessSet::DoWork() { ++nfds; } - interrupted_ = false; + interrupted_ = 0; int ret = ppoll(&fds.front(), nfds, NULL, &old_mask_); if (ret == -1) { if (errno != EINTR) { perror("ninja: ppoll"); return false; } - return interrupted_; + return IsInterrupted(); } + HandlePendingInterruption(); + if (IsInterrupted()) + return true; + nfds_t cur_nfd = 0; for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ) { @@ -227,7 +254,7 @@ bool SubprocessSet::DoWork() { ++i; } - return interrupted_; + return IsInterrupted(); } #else // !defined(USE_PPOLL) @@ -246,16 +273,20 @@ bool SubprocessSet::DoWork() { } } - interrupted_ = false; + interrupted_ = 0; int ret = pselect(nfds, &set, 0, 0, 0, &old_mask_); if (ret == -1) { if (errno != EINTR) { perror("ninja: pselect"); return false; } - return interrupted_; + return IsInterrupted(); } + HandlePendingInterruption(); + if (IsInterrupted()) + return true; + for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ) { int fd = (*i)->fd_; @@ -270,7 +301,7 @@ bool SubprocessSet::DoWork() { ++i; } - return interrupted_; + return IsInterrupted(); } #endif // !defined(USE_PPOLL) @@ -285,10 +316,10 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) - // Since the foreground process is in our process group, it will receive a - // SIGINT at the same time as us. + // Since the foreground process is in our process group, it will receive + // the interruption signal (i.e. SIGINT or SIGTERM) at the same time as us. if (!(*i)->use_console_) - kill(-(*i)->pid_, SIGINT); + kill(-(*i)->pid_, interrupted_); for (vector<Subprocess*>::iterator i = running_.begin(); i != running_.end(); ++i) delete *i; diff --git a/src/subprocess.h b/src/subprocess.h index b7a1a4c..a001fc9 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -89,9 +89,15 @@ struct SubprocessSet { static HANDLE ioport_; #else static void SetInterruptedFlag(int signum); - static bool interrupted_; + static void HandlePendingInterruption(); + /// Store the signal number that causes the interruption. + /// 0 if not interruption. + static int interrupted_; - struct sigaction old_act_; + static bool IsInterrupted() { return interrupted_ != 0; } + + struct sigaction old_int_act_; + struct sigaction old_term_act_; sigset_t old_mask_; #endif }; diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 775a13a..07cc52f 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -16,8 +16,11 @@ #include "test.h" +#include <string> + #ifndef _WIN32 // SetWithLots need setrlimit. +#include <stdio.h> #include <sys/time.h> #include <sys/resource.h> #include <unistd.h> @@ -92,15 +95,50 @@ TEST_F(SubprocessTest, InterruptParent) { return; } - ADD_FAILURE() << "We should have been interrupted"; + ASSERT_FALSE("We should have been interrupted"); +} + +TEST_F(SubprocessTest, InterruptChildWithSigTerm) { + Subprocess* subproc = subprocs_.Add("kill -TERM $$"); + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { + subprocs_.DoWork(); + } + + EXPECT_EQ(ExitInterrupted, subproc->Finish()); +} + +TEST_F(SubprocessTest, InterruptParentWithSigTerm) { + Subprocess* subproc = subprocs_.Add("kill -TERM $PPID ; sleep 1"); + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { + bool interrupted = subprocs_.DoWork(); + if (interrupted) + return; + } + + ASSERT_FALSE("We should have been interrupted"); } +// A shell command to check if the current process is connected to a terminal. +// This is different from having stdin/stdout/stderr be a terminal. (For +// instance consider the command "yes < /dev/null > /dev/null 2>&1". +// As "ps" will confirm, "yes" could still be connected to a terminal, despite +// not having any of the standard file descriptors be a terminal. +static const char kIsConnectedToTerminal[] = "tty < /dev/tty > /dev/null"; + TEST_F(SubprocessTest, Console) { // Skip test if we don't have the console ourselves. if (isatty(0) && isatty(1) && isatty(2)) { - Subprocess* subproc = subprocs_.Add("test -t 0 -a -t 1 -a -t 2", - /*use_console=*/true); - ASSERT_NE((Subprocess *) 0, subproc); + // Test that stdin, stdout and stderr are a terminal. + // Also check that the current process is connected to a terminal. + Subprocess* subproc = + subprocs_.Add(std::string("test -t 0 -a -t 1 -a -t 2 && ") + + std::string(kIsConnectedToTerminal), + /*use_console=*/true); + ASSERT_NE((Subprocess*)0, subproc); while (!subproc->Done()) { subprocs_.DoWork(); @@ -110,6 +148,18 @@ TEST_F(SubprocessTest, Console) { } } +TEST_F(SubprocessTest, NoConsole) { + Subprocess* subproc = + subprocs_.Add(kIsConnectedToTerminal, /*use_console=*/false); + ASSERT_NE((Subprocess*)0, subproc); + + while (!subproc->Done()) { + subprocs_.DoWork(); + } + + EXPECT_NE(ExitSuccess, subproc->Finish()); +} + #endif TEST_F(SubprocessTest, SetWithSingle) { @@ -171,14 +221,15 @@ TEST_F(SubprocessTest, SetWithMulti) { TEST_F(SubprocessTest, SetWithLots) { // Arbitrary big number; needs to be over 1024 to confirm we're no longer // hostage to pselect. - const size_t kNumProcs = 1025; + const unsigned kNumProcs = 1025; // Make sure [ulimit -n] isn't going to stop us from working. rlimit rlim; ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlim)); - ASSERT_GT(rlim.rlim_cur, kNumProcs) - << "Raise [ulimit -n] well above " << kNumProcs - << " to make this test go"; + if (rlim.rlim_cur < kNumProcs) { + printf("Raise [ulimit -n] well above %u (currently %lu) to make this test go\n", kNumProcs, rlim.rlim_cur); + return; + } vector<Subprocess*> procs; for (size_t i = 0; i < kNumProcs; ++i) { @@ -194,7 +245,7 @@ TEST_F(SubprocessTest, SetWithLots) { } ASSERT_EQ(kNumProcs, subprocs_.finished_.size()); } -#endif // !__APPLE__ && !_WIN32 +#endif // !__APPLE__ && !_WIN32 // TODO: this test could work on Windows, just not sure how to simply // read stdin. diff --git a/src/test.cc b/src/test.cc index 45a9226..aed8db7 100644 --- a/src/test.cc +++ b/src/test.cc @@ -12,20 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifdef _WIN32 +#include <direct.h> // Has to be before util.h is included. +#endif + #include "test.h" #include <algorithm> #include <errno.h> +#ifdef _WIN32 +#include <windows.h> +#else +#include <unistd.h> +#endif #include "build_log.h" +#include "graph.h" #include "manifest_parser.h" #include "util.h" -#ifdef _WIN32 -#include <windows.h> -#endif - namespace { #ifdef _WIN32 @@ -84,20 +90,54 @@ void StateTestWithBuiltinRules::AddCatRule(State* state) { } Node* StateTestWithBuiltinRules::GetNode(const string& path) { - return state_.GetNode(path); + EXPECT_FALSE(strpbrk(path.c_str(), "/\\")); + return state_.GetNode(path, 0); } void AssertParse(State* state, const char* input) { ManifestParser parser(state, NULL); string err; - ASSERT_TRUE(parser.ParseTest(input, &err)) << err; + EXPECT_TRUE(parser.ParseTest(input, &err)); ASSERT_EQ("", err); + VerifyGraph(*state); } void AssertHash(const char* expected, uint64_t actual) { ASSERT_EQ(BuildLog::LogEntry::HashCommand(expected), actual); } +void VerifyGraph(const State& state) { + for (vector<Edge*>::const_iterator e = state.edges_.begin(); + e != state.edges_.end(); ++e) { + // All edges need at least one output. + EXPECT_FALSE((*e)->outputs_.empty()); + // Check that the edge's inputs have the edge as out-edge. + for (vector<Node*>::const_iterator in_node = (*e)->inputs_.begin(); + in_node != (*e)->inputs_.end(); ++in_node) { + const vector<Edge*>& out_edges = (*in_node)->out_edges(); + EXPECT_NE(std::find(out_edges.begin(), out_edges.end(), *e), + out_edges.end()); + } + // Check that the edge's outputs have the edge as in-edge. + for (vector<Node*>::const_iterator out_node = (*e)->outputs_.begin(); + out_node != (*e)->outputs_.end(); ++out_node) { + EXPECT_EQ((*out_node)->in_edge(), *e); + } + } + + // The union of all in- and out-edges of each nodes should be exactly edges_. + set<const Edge*> node_edge_set; + for (State::Paths::const_iterator p = state.paths_.begin(); + p != state.paths_.end(); ++p) { + const Node* n = p->second; + if (n->in_edge()) + node_edge_set.insert(n->in_edge()); + node_edge_set.insert(n->out_edges().begin(), n->out_edges().end()); + } + set<const Edge*> edge_set(state.edges_.begin(), state.edges_.end()); + EXPECT_EQ(node_edge_set, edge_set); +} + void VirtualFileSystem::Create(const string& path, const string& contents) { files_[path].mtime = now_; @@ -105,10 +145,12 @@ void VirtualFileSystem::Create(const string& path, files_created_.insert(path); } -TimeStamp VirtualFileSystem::Stat(const string& path) { - FileMap::iterator i = files_.find(path); - if (i != files_.end()) +TimeStamp VirtualFileSystem::Stat(const string& path, string* err) const { + FileMap::const_iterator i = files_.find(path); + if (i != files_.end()) { + *err = i->second.stat_error; return i->second.mtime; + } return 0; } @@ -15,12 +15,94 @@ #ifndef NINJA_TEST_H_ #define NINJA_TEST_H_ -#include <gtest/gtest.h> - #include "disk_interface.h" #include "state.h" #include "util.h" +// A tiny testing framework inspired by googletest, but much simpler and +// faster to compile. It supports most things commonly used from googltest. The +// most noticeable things missing: EXPECT_* and ASSERT_* don't support +// streaming notes to them with operator<<, and for failing tests the lhs and +// rhs are not printed. That's so that this header does not have to include +// sstream, which slows down building ninja_test almost 20%. +namespace testing { +class Test { + bool failed_; + int assertion_failures_; + public: + Test() : failed_(false), assertion_failures_(0) {} + virtual ~Test() {} + virtual void SetUp() {} + virtual void TearDown() {} + virtual void Run() = 0; + + bool Failed() const { return failed_; } + int AssertionFailures() const { return assertion_failures_; } + void AddAssertionFailure() { assertion_failures_++; } + bool Check(bool condition, const char* file, int line, const char* error); +}; +} + +void RegisterTest(testing::Test* (*)(), const char*); + +extern testing::Test* g_current_test; +#define TEST_F_(x, y, name) \ + struct y : public x { \ + static testing::Test* Create() { return g_current_test = new y; } \ + virtual void Run(); \ + }; \ + struct Register##y { \ + Register##y() { RegisterTest(y::Create, name); } \ + }; \ + Register##y g_register_##y; \ + void y::Run() + +#define TEST_F(x, y) TEST_F_(x, x##y, #x "." #y) +#define TEST(x, y) TEST_F_(testing::Test, x##y, #x "." #y) + +#define EXPECT_EQ(a, b) \ + g_current_test->Check(a == b, __FILE__, __LINE__, #a " == " #b) +#define EXPECT_NE(a, b) \ + g_current_test->Check(a != b, __FILE__, __LINE__, #a " != " #b) +#define EXPECT_GT(a, b) \ + g_current_test->Check(a > b, __FILE__, __LINE__, #a " > " #b) +#define EXPECT_LT(a, b) \ + g_current_test->Check(a < b, __FILE__, __LINE__, #a " < " #b) +#define EXPECT_GE(a, b) \ + g_current_test->Check(a >= b, __FILE__, __LINE__, #a " >= " #b) +#define EXPECT_LE(a, b) \ + g_current_test->Check(a <= b, __FILE__, __LINE__, #a " <= " #b) +#define EXPECT_TRUE(a) \ + g_current_test->Check(static_cast<bool>(a), __FILE__, __LINE__, #a) +#define EXPECT_FALSE(a) \ + g_current_test->Check(!static_cast<bool>(a), __FILE__, __LINE__, #a) + +#define ASSERT_EQ(a, b) \ + if (!EXPECT_EQ(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_NE(a, b) \ + if (!EXPECT_NE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_GT(a, b) \ + if (!EXPECT_GT(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_LT(a, b) \ + if (!EXPECT_LT(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_GE(a, b) \ + if (!EXPECT_GE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_LE(a, b) \ + if (!EXPECT_LE(a, b)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_TRUE(a) \ + if (!EXPECT_TRUE(a)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_FALSE(a) \ + if (!EXPECT_FALSE(a)) { g_current_test->AddAssertionFailure(); return; } +#define ASSERT_NO_FATAL_FAILURE(a) \ + { \ + int f = g_current_test->AssertionFailures(); \ + a; \ + if (f != g_current_test->AssertionFailures()) { \ + g_current_test->AddAssertionFailure(); \ + return; \ + } \ + } + // Support utilites for tests. struct Node; @@ -42,6 +124,7 @@ struct StateTestWithBuiltinRules : public testing::Test { void AssertParse(State* state, const char* input); void AssertHash(const char* expected, uint64_t actual); +void VerifyGraph(const State& state); /// An implementation of DiskInterface that uses an in-memory representation /// of disk state. It also logs file accesses and directory creations @@ -59,7 +142,7 @@ struct VirtualFileSystem : public DiskInterface { } // DiskInterface - virtual TimeStamp Stat(const string& path); + virtual TimeStamp Stat(const string& path, string* err) const; virtual bool WriteFile(const string& path, const string& contents); virtual bool MakeDir(const string& path); virtual string ReadFile(const string& path, string* err); @@ -68,6 +151,7 @@ struct VirtualFileSystem : public DiskInterface { /// An entry for a single in-memory file. struct Entry { int mtime; + string stat_error; // If mtime is -1. string contents; }; diff --git a/src/util.cc b/src/util.cc index 484b0c1..aa47f2f 100644 --- a/src/util.cc +++ b/src/util.cc @@ -14,7 +14,10 @@ #include "util.h" -#ifdef _WIN32 +#ifdef __CYGWIN__ +#include <windows.h> +#include <io.h> +#elif defined( _WIN32) #include <windows.h> #include <io.h> #include <share.h> @@ -85,19 +88,33 @@ void Error(const char* msg, ...) { fprintf(stderr, "\n"); } -bool CanonicalizePath(string* path, string* err) { +bool CanonicalizePath(string* path, unsigned int* slash_bits, string* err) { METRIC_RECORD("canonicalize str"); size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; - if (!CanonicalizePath(str, &len, err)) + if (!CanonicalizePath(str, &len, slash_bits, err)) return false; path->resize(len); return true; } -bool CanonicalizePath(char* path, size_t* len, string* err) { +#ifdef _WIN32 +static unsigned int ShiftOverBit(int offset, unsigned int bits) { + // e.g. for |offset| == 2: + // | ... 9 8 7 6 5 4 3 2 1 0 | + // \_________________/ \_/ + // above below + // So we drop the bit at offset and move above "down" into its place. + unsigned int above = bits & ~((1 << (offset + 1)) - 1); + unsigned int below = bits & ((1 << offset) - 1); + return (above >> 1) | below; +} +#endif + +bool CanonicalizePath(char* path, size_t* len, unsigned int* slash_bits, + string* err) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. METRIC_RECORD("canonicalize path"); @@ -115,12 +132,37 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { const char* src = start; const char* end = start + *len; +#ifdef _WIN32 + unsigned int bits = 0; + unsigned int bits_mask = 1; + int bits_offset = 0; + // Convert \ to /, setting a bit in |bits| for each \ encountered. + for (char* c = path; c < end; ++c) { + switch (*c) { + case '\\': + bits |= bits_mask; + *c = '/'; + // Intentional fallthrough. + case '/': + bits_mask <<= 1; + bits_offset++; + } + } + if (bits_offset > 32) { + *err = "too many path components"; + return false; + } + bits_offset = 0; +#endif + if (*src == '/') { #ifdef _WIN32 + bits_offset++; // network path starts with // if (*len > 1 && *(src + 1) == '/') { src += 2; dst += 2; + bits_offset++; } else { ++src; ++dst; @@ -136,6 +178,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { if (src + 1 == end || src[1] == '/') { // '.' component; eliminate. src += 2; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); +#endif continue; } else if (src[1] == '.' && (src + 2 == end || src[2] == '/')) { // '..' component. Back up if possible. @@ -143,6 +188,11 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { dst = components[component_count - 1]; src += 3; --component_count; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); + bits_offset--; + bits = ShiftOverBit(bits_offset, bits); +#endif } else { *dst++ = *src++; *dst++ = *src++; @@ -154,6 +204,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { if (*src == '/') { src++; +#ifdef _WIN32 + bits = ShiftOverBit(bits_offset, bits); +#endif continue; } @@ -164,6 +217,9 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { while (*src != '/' && src != end) *dst++ = *src++; +#ifdef _WIN32 + bits_offset++; +#endif *dst++ = *src++; // Copy '/' or final \0 character as well. } @@ -173,6 +229,11 @@ bool CanonicalizePath(char* path, size_t* len, string* err) { } *len = dst - start - 1; +#ifdef _WIN32 + *slash_bits = bits; +#else + *slash_bits = 0; +#endif return true; } @@ -280,7 +341,38 @@ void GetWin32EscapedString(const string& input, string* result) { } int ReadFile(const string& path, string* contents, string* err) { - FILE* f = fopen(path.c_str(), "r"); +#ifdef _WIN32 + // This makes a ninja run on a set of 1500 manifest files about 4% faster + // than using the generic fopen code below. + err->clear(); + HANDLE f = ::CreateFile(path.c_str(), + GENERIC_READ, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + FILE_FLAG_SEQUENTIAL_SCAN, + NULL); + if (f == INVALID_HANDLE_VALUE) { + err->assign(GetLastErrorString()); + return -ENOENT; + } + + for (;;) { + DWORD len; + char buf[64 << 10]; + if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { + err->assign(GetLastErrorString()); + contents->clear(); + return -1; + } + if (len == 0) + break; + contents->append(buf, len); + } + ::CloseHandle(f); + return 0; +#else + FILE* f = fopen(path.c_str(), "rb"); if (!f) { err->assign(strerror(errno)); return -errno; @@ -299,6 +391,7 @@ int ReadFile(const string& path, string* contents, string* err) { } fclose(f); return 0; +#endif } void SetCloseOnExec(int fd) { @@ -407,7 +500,7 @@ string StripAnsiEscapeCodes(const string& in) { int GetProcessorCount() { #ifdef _WIN32 SYSTEM_INFO info; - GetSystemInfo(&info); + GetNativeSystemInfo(&info); return info.dwNumberOfProcessors; #else return sysconf(_SC_NPROCESSORS_ONLN); @@ -415,10 +508,70 @@ int GetProcessorCount() { } #if defined(_WIN32) || defined(__CYGWIN__) +static double CalculateProcessorLoad(uint64_t idle_ticks, uint64_t total_ticks) +{ + static uint64_t previous_idle_ticks = 0; + static uint64_t previous_total_ticks = 0; + static double previous_load = -0.0; + + uint64_t idle_ticks_since_last_time = idle_ticks - previous_idle_ticks; + uint64_t total_ticks_since_last_time = total_ticks - previous_total_ticks; + + bool first_call = (previous_total_ticks == 0); + bool ticks_not_updated_since_last_call = (total_ticks_since_last_time == 0); + + double load; + if (first_call || ticks_not_updated_since_last_call) { + load = previous_load; + } else { + // Calculate load. + double idle_to_total_ratio = + ((double)idle_ticks_since_last_time) / total_ticks_since_last_time; + double load_since_last_call = 1.0 - idle_to_total_ratio; + + // Filter/smooth result when possible. + if(previous_load > 0) { + load = 0.9 * previous_load + 0.1 * load_since_last_call; + } else { + load = load_since_last_call; + } + } + + previous_load = load; + previous_total_ticks = total_ticks; + previous_idle_ticks = idle_ticks; + + return load; +} + +static uint64_t FileTimeToTickCount(const FILETIME & ft) +{ + uint64_t high = (((uint64_t)(ft.dwHighDateTime)) << 32); + uint64_t low = ft.dwLowDateTime; + return (high | low); +} + double GetLoadAverage() { - // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. - // Remember to also update Usage() when this is fixed. - return -0.0f; + FILETIME idle_time, kernel_time, user_time; + BOOL get_system_time_succeeded = + GetSystemTimes(&idle_time, &kernel_time, &user_time); + + double posix_compatible_load; + if (get_system_time_succeeded) { + uint64_t idle_ticks = FileTimeToTickCount(idle_time); + + // kernel_time from GetSystemTimes already includes idle_time. + uint64_t total_ticks = + FileTimeToTickCount(kernel_time) + FileTimeToTickCount(user_time); + + double processor_load = CalculateProcessorLoad(idle_ticks, total_ticks); + posix_compatible_load = processor_load * GetProcessorCount(); + + } else { + posix_compatible_load = -0.0; + } + + return posix_compatible_load; } #else double GetLoadAverage() { @@ -41,9 +41,11 @@ void Warning(const char* msg, ...); void Error(const char* msg, ...); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". -bool CanonicalizePath(string* path, string* err); - -bool CanonicalizePath(char* path, size_t* len, string* err); +/// |slash_bits| has bits set starting from lowest for a backslash that was +/// normalized to a forward slash. (only used on Windows) +bool CanonicalizePath(string* path, unsigned int* slash_bits, string* err); +bool CanonicalizePath(char* path, size_t* len, unsigned int* slash_bits, + string* err); /// Appends |input| to |*result|, escaping according to the whims of either /// Bash, or Win32's CommandLineToArgvW(). diff --git a/src/util_test.cc b/src/util_test.cc index b58d15e..8ca7f56 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -16,6 +16,15 @@ #include "test.h" +namespace { + +bool CanonicalizePath(string* path, string* err) { + unsigned int unused; + return ::CanonicalizePath(path, &unused, err); +} + +} // namespace + TEST(CanonicalizePath, PathSamples) { string path; string err; @@ -84,6 +93,201 @@ TEST(CanonicalizePath, PathSamples) { EXPECT_EQ("", path); } +#ifdef _WIN32 +TEST(CanonicalizePath, PathSamplesWindows) { + string path; + string err; + + EXPECT_FALSE(CanonicalizePath(&path, &err)); + EXPECT_EQ("empty path", err); + + path = "foo.h"; err = ""; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo.h", path); + + path = ".\\foo\\.\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar.h", path); + + path = ".\\x\\foo\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("x/bar.h", path); + + path = ".\\x\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar.h", path); + + path = "foo\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/bar", path); + + path = "foo\\\\.\\\\..\\\\\\bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("bar", path); + + path = ".\\x\\..\\foo\\..\\..\\bar.h"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("../bar.h", path); + + path = "foo\\.\\."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\bar\\.."; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo", path); + + path = "foo\\.hidden_bar"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("foo/.hidden_bar", path); + + path = "\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("/foo", path); + + path = "\\\\foo"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("//foo", path); + + path = "\\"; + EXPECT_TRUE(CanonicalizePath(&path, &err)); + EXPECT_EQ("", path); +} + +TEST(CanonicalizePath, SlashTracking) { + string path; + string err; + unsigned int slash_bits; + + path = "foo.h"; err = ""; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a/bcd/efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(4, slash_bits); + + path = "a\\bcd/efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(5, slash_bits); + + path = "a\\bcd\\efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(7, slash_bits); + + path = "a/bcd/efh/foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/bcd/efh/foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\./efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/efh/foo.h", path); + EXPECT_EQ(3, slash_bits); + + path = "a\\../efh\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("efh/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b\\c\\d\\e\\f\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/b/c/d/e/f/g/foo.h", path); + EXPECT_EQ(127, slash_bits); + + path = "a\\b\\c\\..\\..\\..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b/c\\../../..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\b/c\\./../..\\g\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/g/foo.h", path); + EXPECT_EQ(3, slash_bits); + + path = "a\\b/c\\./../..\\g/foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/g/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a\\\\\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); + + path = "a/\\\\foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(0, slash_bits); + + path = "a\\//foo.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ("a/foo.h", path); + EXPECT_EQ(1, slash_bits); +} + +TEST(CanonicalizePath, CanonicalizeNotExceedingLen) { + // Make sure searching \/ doesn't go past supplied len. + char buf[] = "foo/bar\\baz.h\\"; // Last \ past end. + unsigned int slash_bits; + string err; + size_t size = 13; + EXPECT_TRUE(::CanonicalizePath(buf, &size, &slash_bits, &err)); + EXPECT_EQ(0, strncmp("foo/bar/baz.h", buf, size)); + EXPECT_EQ(2, slash_bits); // Not including the trailing one. +} + +TEST(CanonicalizePath, TooManyComponents) { + string path; + string err; + unsigned int slash_bits; + + // 32 is OK. + path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + + // Backslashes version. + path = + "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\." + "\\a\\.\\a\\.\\a\\.\\a\\.\\x.h"; + EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(slash_bits, 0xffff); + + // 33 is not. + err = ""; + path = + "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/x.h"; + EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(err, "too many path components"); + + // Backslashes version. + err = ""; + path = + "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\." + "\\a\\.\\a\\.\\a\\.\\a\\.\\a\\x.h"; + EXPECT_FALSE(CanonicalizePath(&path, &slash_bits, &err)); + EXPECT_EQ(err, "too many path components"); +} +#endif + TEST(CanonicalizePath, EmptyResult) { string path; string err; @@ -122,26 +326,27 @@ TEST(CanonicalizePath, NotNullTerminated) { string path; string err; size_t len; + unsigned int unused; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); path = "foo/../file bar/."; len = strlen("foo/../file"); - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &err)); + EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); EXPECT_EQ(strlen("file"), len); EXPECT_EQ("file ./file bar/.", string(path)); } TEST(PathEscaping, TortureTest) { string result; - + GetWin32EscapedString("foo bar\\\"'$@d!st!c'\\path'\\", &result); EXPECT_EQ("\"foo bar\\\\\\\"'$@d!st!c'\\path'\\\\\"", result); - result.clear(); + result.clear(); GetShellEscapedString("foo bar\"/'$@d!st!c'/path'", &result); EXPECT_EQ("'foo bar\"/'\\''$@d!st!c'\\''/path'\\'''", result); diff --git a/src/version.cc b/src/version.cc index 1406d91..4c2aea1 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,7 @@ #include "util.h" -const char* kNinjaVersion = "1.4.0.git"; +const char* kNinjaVersion = "1.6.0.git"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); |