summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEvan Martin <martine@danga.com>2012-12-29 21:36:00 (GMT)
committerEvan Martin <martine@danga.com>2012-12-29 21:47:41 (GMT)
commit2c953d1501de5195e2485185fa24a2ebfd76bbb5 (patch)
tree2fc88e378a6df571bb125d282b14475f2b9ba05c
parent7096bf1507f98be981aa14ffd9ed5a4a8b1c1494 (diff)
parent3249938cdf574058a066436aea06b0541ded6958 (diff)
downloadNinja-1.1.0.zip
Ninja-1.1.0.tar.gz
Ninja-1.1.0.tar.bz2
version 1.1.0v1.1.0
-rw-r--r--.gitignore5
-rw-r--r--HACKING.md7
-rwxr-xr-xbootstrap.py39
-rwxr-xr-xconfigure.py56
-rw-r--r--doc/manual.asciidoc54
-rw-r--r--misc/bash-completion12
-rw-r--r--misc/ninja-mode.el3
-rw-r--r--misc/ninja.vim12
-rw-r--r--misc/ninja_syntax.py16
-rwxr-xr-xmisc/ninja_test.py6
-rw-r--r--misc/packaging/ninja.spec21
-rwxr-xr-xmisc/packaging/rpmbuild.sh29
-rwxr-xr-xsrc/browse.py77
-rw-r--r--src/build.cc134
-rw-r--r--src/build.h78
-rw-r--r--src/build_log.cc26
-rw-r--r--src/build_log.h4
-rw-r--r--src/build_log_test.cc27
-rw-r--r--src/build_test.cc141
-rw-r--r--src/clean.cc40
-rw-r--r--src/clean.h8
-rw-r--r--src/disk_interface.cc12
-rw-r--r--src/disk_interface_test.cc3
-rw-r--r--src/graph.cc50
-rw-r--r--src/graph.h16
-rw-r--r--src/hash_map.h4
-rw-r--r--src/includes_normalize_test.cc16
-rw-r--r--src/lexer.cc389
-rw-r--r--src/lexer.h1
-rw-r--r--src/lexer.in.cc2
-rw-r--r--src/manifest_parser.cc72
-rw-r--r--src/manifest_parser.h1
-rw-r--r--src/metrics.h22
-rw-r--r--src/msvc_helper-win32.cc26
-rw-r--r--src/msvc_helper.h7
-rw-r--r--src/msvc_helper_main-win32.cc12
-rw-r--r--src/msvc_helper_test.cc39
-rw-r--r--src/ninja.cc137
-rw-r--r--src/state.cc64
-rw-r--r--src/state.h63
-rw-r--r--src/state_test.cc2
-rw-r--r--src/subprocess-posix.cc2
-rw-r--r--src/subprocess-win32.cc7
-rw-r--r--src/subprocess_test.cc18
-rw-r--r--src/util.cc2
-rw-r--r--src/util.h3
46 files changed, 1239 insertions, 526 deletions
diff --git a/.gitignore b/.gitignore
index 19a08ee..3cee921 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,8 @@ TAGS
/doc/manual.html
/doc/doxygen
/gtest-1.6.0
+
+# Eclipse project files
+.project
+.cproject
+
diff --git a/HACKING.md b/HACKING.md
index a777b57..885d2d7 100644
--- a/HACKING.md
+++ b/HACKING.md
@@ -159,8 +159,15 @@ it's locked while in use.
### Via mingw on Linux (not well supported)
+Setup on Ubuntu Lucid:
* `sudo apt-get install gcc-mingw32 wine`
* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar`
+
+Setup on Ubuntu Precise:
+* `sudo apt-get install gcc-mingw-w64-i686 g++-mingw-w64-i686 wine`
+* `export CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ AR=i686-w64-mingw32-ar`
+
+Then run:
* `./configure.py --platform=mingw --host=linux`
* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja`
* Run: `./ninja.exe` (implicitly runs through wine(!))
diff --git a/bootstrap.py b/bootstrap.py
index 3032a9b..a847df9 100755
--- a/bootstrap.py
+++ b/bootstrap.py
@@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+from __future__ import print_function
+
from optparse import OptionParser
import sys
import os
@@ -29,6 +31,10 @@ parser.add_option('--verbose', action='store_true',
help='enable verbose build',)
parser.add_option('--x64', action='store_true',
help='force 64-bit build (Windows)',)
+# TODO: make this --platform to match configure.py.
+parser.add_option('--windows', action='store_true',
+ help='force native Windows build (when using Cygwin Python)',
+ default=sys.platform.startswith('win32'))
(options, conf_args) = parser.parse_args()
def run(*args, **kwargs):
@@ -44,11 +50,12 @@ if sys.platform.startswith('freebsd'):
cflags.append('-I/usr/local/include')
ldflags.append('-L/usr/local/lib')
-print 'Building ninja manually...'
+print('Building ninja manually...')
try:
os.mkdir('build')
-except OSError, e:
+except OSError:
+ e = sys.exc_info()[1]
if e.errno != errno.EEXIST:
raise
@@ -63,7 +70,7 @@ for src in glob.glob('src/*.cc'):
if filename == 'browse.cc': # Depends on generated header.
continue
- if sys.platform.startswith('win32'):
+ if options.windows:
if src.endswith('-posix.cc'):
continue
else:
@@ -72,7 +79,7 @@ for src in glob.glob('src/*.cc'):
sources.append(src)
-if sys.platform.startswith('win32'):
+if options.windows:
sources.append('src/getopt.c')
vcdir = os.environ.get('VCINSTALLDIR')
@@ -87,14 +94,14 @@ else:
cflags.extend(['-Wno-deprecated',
'-DNINJA_PYTHON="' + sys.executable + '"',
'-DNINJA_BOOTSTRAP'])
- if sys.platform.startswith('win32'):
+ if options.windows:
cflags.append('-D_WIN32_WINNT=0x0501')
if options.x64:
cflags.append('-m64')
args.extend(cflags)
args.extend(ldflags)
binary = 'ninja.bootstrap'
-if sys.platform.startswith('win32'):
+if options.windows:
binary = 'ninja.bootstrap.exe'
args.extend(sources)
if vcdir:
@@ -103,16 +110,20 @@ else:
args.extend(['-o', binary])
if options.verbose:
- print ' '.join(args)
+ print(' '.join(args))
-run(args)
+try:
+ run(args)
+except:
+ print('Failure running:', args)
+ raise
verbose = []
if options.verbose:
verbose = ['-v']
-if sys.platform.startswith('win32'):
- print 'Building ninja using itself...'
+if options.windows:
+ print('Building ninja using itself...')
run([sys.executable, 'configure.py', '--with-ninja=%s' % binary] +
conf_args)
run(['./' + binary] + verbose)
@@ -124,17 +135,17 @@ if sys.platform.startswith('win32'):
for obj in glob.glob('*.obj'):
os.unlink(obj)
- print """
+ print("""
Done!
Note: to work around Windows file locking, where you can't rebuild an
in-use binary, to run ninja after making any changes to build ninja itself
you should run ninja.bootstrap instead. Your build is also configured to
use ninja.bootstrap.exe as the MSVC helper; see the --with-ninja flag of
-the --help output of configure.py."""
+the --help output of configure.py.""")
else:
- print 'Building ninja using itself...'
+ print('Building ninja using itself...')
run([sys.executable, 'configure.py'] + conf_args)
run(['./' + binary] + verbose)
os.unlink(binary)
- print 'Done!'
+ print('Done!')
diff --git a/configure.py b/configure.py
index 98274e6..9391a68 100755
--- a/configure.py
+++ b/configure.py
@@ -19,6 +19,8 @@
Projects that use ninja themselves should either write a similar script
or use a meta-build system that supports Ninja output."""
+from __future__ import print_function
+
from optparse import OptionParser
import os
import sys
@@ -50,7 +52,7 @@ parser.add_option('--with-ninja', metavar='NAME',
default="ninja")
(options, args) = parser.parse_args()
if args:
- print 'ERROR: extra unparsed command-line arguments:', args
+ print('ERROR: extra unparsed command-line arguments:', args)
sys.exit(1)
platform = options.platform
@@ -140,6 +142,7 @@ else:
'-fno-rtti',
'-fno-exceptions',
'-fvisibility=hidden', '-pipe',
+ '-Wno-missing-field-initializers',
'-DNINJA_PYTHON="%s"' % options.with_python]
if options.debug:
cflags += ['-D_GLIBCXX_DEBUG', '-D_GLIBCXX_DEBUG_PEDANTIC']
@@ -168,7 +171,9 @@ else:
libs.append('-lprofiler')
def shell_escape(str):
- """Escape str such that it's interpreted as a single argument by the shell."""
+ """Escape str such that it's interpreted as a single argument by
+ the shell."""
+
# This isn't complete, but it's just enough to make NINJA_PYTHON work.
if platform in ('windows', 'mingw'):
return str
@@ -255,7 +260,7 @@ if has_re2c():
n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
else:
- print ("warning: A compatible version of re2c (>= 0.11.3) was not found; "
+ print("warning: A compatible version of re2c (>= 0.11.3) was not found; "
"changes to src/*.in.cc will not affect your build.")
n.newline()
@@ -277,11 +282,12 @@ for name in ['build',
'util']:
objs += cxx(name)
if platform in ('mingw', 'windows'):
- objs += cxx('subprocess-win32')
+ for name in ['subprocess-win32',
+ 'includes_normalize-win32',
+ 'msvc_helper-win32',
+ 'msvc_helper_main-win32']:
+ objs += cxx(name)
if platform == 'windows':
- objs += cxx('includes_normalize-win32')
- objs += cxx('msvc_helper-win32')
- objs += cxx('msvc_helper_main-win32')
objs += cxx('minidump-win32')
objs += cc('getopt')
else:
@@ -309,7 +315,7 @@ all_targets += ninja
n.comment('Tests all build into ninja_test executable.')
variables = []
-test_cflags = None
+test_cflags = cflags[:]
test_ldflags = None
test_libs = libs
objs = []
@@ -328,13 +334,17 @@ if options.with_gtest:
os.path.join(path, 'src', 'gtest_main.cc'),
variables=[('cflags', gtest_cflags)])
- test_cflags = cflags + ['-DGTEST_HAS_RTTI=0',
- '-I%s' % os.path.join(path, 'include')]
+ test_cflags.append('-I%s' % os.path.join(path, 'include'))
elif platform == 'windows':
test_libs.extend(['gtest_main.lib', 'gtest.lib'])
else:
+ test_cflags.append('-DGTEST_HAS_RTTI=0')
test_libs.extend(['-lgtest_main', '-lgtest'])
+if test_cflags == cflags:
+ test_cflags = None
+
+n.variable('test_cflags', test_cflags)
for name in ['build_log_test',
'build_test',
'clean_test',
@@ -348,8 +358,8 @@ for name in ['build_log_test',
'subprocess_test',
'test',
'util_test']:
- objs += cxx(name, variables=[('cflags', test_cflags)])
-if platform == 'windows':
+ objs += cxx(name, variables=[('cflags', '$test_cflags')])
+if platform in ('windows', 'mingw'):
for name in ['includes_normalize_test', 'msvc_helper_test']:
objs += cxx(name, variables=[('cflags', test_cflags)])
@@ -362,7 +372,7 @@ n.newline()
all_targets += ninja_test
-n.comment('Ancilliary executables.')
+n.comment('Ancillary executables.')
objs = cxx('parser_perftest')
all_targets += n.build(binary('parser_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
@@ -427,23 +437,11 @@ n.newline()
if host == 'linux':
n.comment('Packaging')
n.rule('rpmbuild',
- command="rpmbuild \
- --define 'ver git' \
- --define \"rel `git rev-parse --short HEAD`\" \
- --define '_topdir %(pwd)/rpm-build' \
- --define '_builddir %{_topdir}' \
- --define '_rpmdir %{_topdir}' \
- --define '_srcrpmdir %{_topdir}' \
- --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \
- --define '_specdir %{_topdir}' \
- --define '_sourcedir %{_topdir}' \
- --quiet \
- -bb misc/packaging/ninja.spec",
- description='Building RPM..')
- n.build('rpm', 'rpmbuild',
- implicit=['ninja','README', 'COPYING', doc('manual.html')])
+ command="misc/packaging/rpmbuild.sh",
+ description='Building rpms..')
+ n.build('rpm', 'rpmbuild')
n.newline()
n.build('all', 'phony', all_targets)
-print 'wrote %s.' % BUILD_FILENAME
+print('wrote %s.' % BUILD_FILENAME)
diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc
index 03d27df..42e5452 100644
--- a/doc/manual.asciidoc
+++ b/doc/manual.asciidoc
@@ -215,6 +215,7 @@ Ninja supports one environment variable to control its behavior.
Several placeholders are available:
* `%s`: The number of started edges.
* `%t`: The total number of edges that must be run to complete the build.
+* `%p`: The percentage of started edges.
* `%r`: The number of currently running edges.
* `%u`: The number of remaining edges to start.
* `%f`: The number of finished edges.
@@ -420,6 +421,59 @@ If the top-level Ninja file is specified as an output of any build
statement and it is out of date, Ninja will rebuild and reload it
before building the targets requested by the user.
+Pools
+~~~~~
+
+Pools allow you to allocate one or more rules or edges a finite number
+of concurrent jobs which is more tightly restricted than the default
+parallelism.
+
+This can be useful, for example, to restrict a particular expensive rule
+(like link steps for huge executables), or to restrict particular build
+statements which you know perform poorly when run concurrently.
+
+Each pool has a `depth` variable which is specified in the build file.
+The pool is then referred to with the `pool` variable on either a rule
+or a build statement.
+
+No matter what pools you specify, ninja will never run more concurrent jobs
+than the default parallelism, or the number of jobs specified on the command
+line (with -j).
+
+----------------
+# No more than 4 links at a time.
+pool link_pool
+ depth = 4
+
+# No more than 1 heavy object at a time.
+pool heavy_object_pool
+ depth = 1
+
+rule link
+ ...
+ pool = link_pool
+
+rule cc
+ ...
+
+# The link_pool is used here. Only 4 links will run concurrently.
+build foo.exe: link input.obj
+
+# A build statement can be exempted from its rule's pool by setting an
+# empty pool. This effectively puts the build statement back into the default
+# pool, which has infinite depth.
+build other.exe: link input.obj
+ pool =
+
+# A build statement can specify a pool directly.
+# Only one of these builds will run at a time.
+build heavy_object1.obj: cc heavy_obj1.cc
+ pool = heavy_object_pool
+build heavy_object2.obj: cc heavy_obj2.cc
+ pool = heavy_object_pool
+
+----------------
+
Generating Ninja files from code
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/misc/bash-completion b/misc/bash-completion
index ac4d051..b40136e 100644
--- a/misc/bash-completion
+++ b/misc/bash-completion
@@ -16,9 +16,17 @@
# . path/to/ninja/misc/bash-completion
_ninja_target() {
- local cur targets
+ local cur targets dir line targets_command OPTIND
cur="${COMP_WORDS[COMP_CWORD]}"
- targets=$((ninja -t targets all 2>/dev/null) | awk -F: '{print $1}')
+ dir="."
+ line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
+ while getopts C: opt "${line[@]}"; do
+ case $opt in
+ C) dir="$OPTARG" ;;
+ esac
+ done;
+ targets_command="ninja -C ${dir} -t targets all"
+ targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
COMPREPLY=($(compgen -W "$targets" -- "$cur"))
return 0
}
diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el
index 44fc82b..d939206 100644
--- a/misc/ninja-mode.el
+++ b/misc/ninja-mode.el
@@ -18,7 +18,8 @@
(setq ninja-keywords
(list
'("^#.*" . font-lock-comment-face)
- (cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include")
+ (cons (concat "^" (regexp-opt '("rule" "build" "subninja" "include"
+ "pool" "default")
'words))
font-lock-keyword-face)
'("\\([[:alnum:]_]+\\) =" . (1 font-lock-variable-name-face))
diff --git a/misc/ninja.vim b/misc/ninja.vim
index 6f0e48d..841902f 100644
--- a/misc/ninja.vim
+++ b/misc/ninja.vim
@@ -1,8 +1,8 @@
" ninja build file syntax.
" Language: ninja build file as described at
" http://martine.github.com/ninja/manual.html
-" Version: 1.2
-" Last Change: 2012/06/01
+" Version: 1.3
+" Last Change: 2012/12/14
" Maintainer: Nicolas Weber <nicolasweber@gmx.de>
" Version 1.2 of this script is in the upstream vim repository and will be
" included in the next vim release. If you change this, please send your change
@@ -25,6 +25,7 @@ syn match ninjaComment /#.*/ contains=@Spell
" lexer.in.cc, ReadToken() and manifest_parser.cc, Parse()
syn match ninjaKeyword "^build\>"
syn match ninjaKeyword "^rule\>"
+syn match ninjaKeyword "^pool\>"
syn match ninjaKeyword "^default\>"
syn match ninjaKeyword "^include\>"
syn match ninjaKeyword "^subninja\>"
@@ -35,7 +36,11 @@ syn match ninjaKeyword "^subninja\>"
" let assignments.
" manifest_parser.cc, ParseRule()
syn region ninjaRule start="^rule" end="^\ze\S" contains=ALL transparent
-syn keyword ninjaRuleCommand contained command depfile description generator restat
+syn keyword ninjaRuleCommand contained command depfile description generator
+ \ pool restat rspfile rspfile_content
+
+syn region ninjaPool start="^pool" end="^\ze\S" contains=ALL transparent
+syn keyword ninjaPoolCommand contained depth
" Strings are parsed as follows:
" lexer.in.cc, ReadEvalString()
@@ -61,6 +66,7 @@ syn match ninjaOperator "\(=\|:\||\|||\)\ze\s"
hi def link ninjaComment Comment
hi def link ninjaKeyword Keyword
hi def link ninjaRuleCommand Statement
+hi def link ninjaPoolCommand Statement
hi def link ninjaWrapLineOperator ninjaOperator
hi def link ninjaOperator Operator
hi def link ninjaSimpleVar ninjaVar
diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py
index 66babbe..ece7eb5 100644
--- a/misc/ninja_syntax.py
+++ b/misc/ninja_syntax.py
@@ -32,8 +32,13 @@ class Writer(object):
value = ' '.join(filter(None, value)) # Filter out empty strings.
self._line('%s = %s' % (key, value), indent)
+ def pool(self, name, depth):
+ self._line('pool %s' % name)
+ self.variable('depth', depth, indent=1)
+
def rule(self, name, command, description=None, depfile=None,
- generator=False, restat=False, rspfile=None, rspfile_content=None):
+ generator=False, pool=None, restat=False, rspfile=None,
+ rspfile_content=None):
self._line('rule %s' % name)
self.variable('command', command, indent=1)
if description:
@@ -42,6 +47,8 @@ class Writer(object):
self.variable('depfile', depfile, indent=1)
if generator:
self.variable('generator', '1', indent=1)
+ if pool:
+ self.variable('pool', pool, indent=1)
if restat:
self.variable('restat', '1', indent=1)
if rspfile:
@@ -65,13 +72,12 @@ class Writer(object):
all_inputs.append('||')
all_inputs.extend(order_only)
- self._line('build %s: %s %s' % (' '.join(out_outputs),
- rule,
- ' '.join(all_inputs)))
+ self._line('build %s: %s' % (' '.join(out_outputs),
+ ' '.join([rule] + all_inputs)))
if variables:
if isinstance(variables, dict):
- iterator = variables.iteritems()
+ iterator = iter(variables.items())
else:
iterator = iter(variables)
diff --git a/misc/ninja_test.py b/misc/ninja_test.py
index b56033e..2aef7ff 100755
--- a/misc/ninja_test.py
+++ b/misc/ninja_test.py
@@ -15,7 +15,11 @@
# limitations under the License.
import unittest
-from StringIO import StringIO
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
import ninja_syntax
diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec
index d513c6d..2f009f6 100644
--- a/misc/packaging/ninja.spec
+++ b/misc/packaging/ninja.spec
@@ -5,6 +5,8 @@ Release: %{rel}%{?dist}
Group: Development/Tools
License: Apache 2.0
URL: https://github.com/martine/ninja
+Source0: %{name}-%{version}-%{release}.tar.gz
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}
%description
Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and
@@ -14,20 +16,25 @@ Ninja joins a sea of other build systems. Its distinguishing goal is to be fast.
which has over 30,000 source files and whose other build systems (including one built from custom non-recursive Makefiles) can take ten
seconds to start building after changing one file. Ninja is under a second.
+%prep
+%setup -q -n %{name}-%{version}-%{release}
+
%build
-# Assuming we've bootstrapped already..
-../ninja manual ninja -C ..
+echo Building..
+./bootstrap.py
+./ninja manual
%install
mkdir -p %{buildroot}%{_bindir} %{buildroot}%{_docdir}
-cp -p ../ninja %{buildroot}%{_bindir}/
-git log --oneline --pretty=format:'%h: %s (%an, %cd)' --abbrev-commit --all > GITLOG
+cp -p ninja %{buildroot}%{_bindir}/
%files
%defattr(-, root, root)
-%doc GITLOG ../COPYING ../README ../doc/manual.html
+%doc COPYING README doc/manual.html
%{_bindir}/*
%clean
-mv %{_topdir}/*.rpm ..
-rm -rf %{_topdir}
+rm -rf %{buildroot}
+
+#The changelog is built automatically from Git history
+%changelog
diff --git a/misc/packaging/rpmbuild.sh b/misc/packaging/rpmbuild.sh
new file mode 100755
index 0000000..9b74c65
--- /dev/null
+++ b/misc/packaging/rpmbuild.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+echo Building ninja RPMs..
+GITROOT=$(git rev-parse --show-toplevel)
+cd $GITROOT
+
+VER=1.0
+REL=$(git rev-parse --short HEAD)git
+RPMTOPDIR=$GITROOT/rpm-build
+echo "Ver: $VER, Release: $REL"
+
+# Create tarball
+mkdir -p $RPMTOPDIR/{SOURCES,SPECS}
+git archive --format=tar --prefix=ninja-${VER}-${REL}/ HEAD | gzip -c > $RPMTOPDIR/SOURCES/ninja-${VER}-${REL}.tar.gz
+
+# Convert git log to RPM's ChangeLog format (shown with rpm -qp --changelog <rpm file>)
+sed -e "s/%{ver}/$VER/" -e "s/%{rel}/$REL/" misc/packaging/ninja.spec > $RPMTOPDIR/SPECS/ninja.spec
+git log --format="* %cd %aN%n- (%h) %s%d%n" --date=local | sed -r 's/[0-9]+:[0-9]+:[0-9]+ //' >> $RPMTOPDIR/SPECS/ninja.spec
+
+# Build SRC and binary RPMs
+rpmbuild --quiet \
+ --define "_topdir $RPMTOPDIR" \
+ --define "_rpmdir $PWD" \
+ --define "_srcrpmdir $PWD" \
+ --define '_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm' \
+ -ba $RPMTOPDIR/SPECS/ninja.spec &&
+
+rm -rf $RPMTOPDIR &&
+echo Done
diff --git a/src/browse.py b/src/browse.py
index 17e67cf..7f15e50 100755
--- a/src/browse.py
+++ b/src/browse.py
@@ -20,7 +20,12 @@ This script is inlined into the final executable and spawned by
it when needed.
"""
-import BaseHTTPServer
+from __future__ import print_function
+
+try:
+ import http.server as httpserver
+except ImportError:
+ import BaseHTTPServer as httpserver
import subprocess
import sys
import webbrowser
@@ -55,12 +60,12 @@ def parse(text):
outputs = []
try:
- target = lines.next()[:-1] # strip trailing colon
+ target = next(lines)[:-1] # strip trailing colon
- line = lines.next()
+ line = next(lines)
(match, rule) = match_strip(line, ' input: ')
if match:
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
while match:
type = None
(match, line) = match_strip(line, '| ')
@@ -70,21 +75,21 @@ def parse(text):
if match:
type = 'order-only'
inputs.append((line, type))
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
match, _ = match_strip(line, ' outputs:')
if match:
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
while match:
outputs.append(line)
- (match, line) = match_strip(lines.next(), ' ')
+ (match, line) = match_strip(next(lines), ' ')
except StopIteration:
pass
return Node(inputs, rule, target, outputs)
-def generate_html(node):
- print '''<!DOCTYPE html>
+def create_page(body):
+ return '''<!DOCTYPE html>
<style>
body {
font-family: sans;
@@ -108,34 +113,42 @@ tt {
.filelist {
-webkit-columns: auto 2;
}
-</style>'''
+</style>
+''' + body
- print '<h1><tt>%s</tt></h1>' % node.target
+def generate_html(node):
+ document = ['<h1><tt>%s</tt></h1>' % node.target]
if node.inputs:
- print '<h2>target is built using rule <tt>%s</tt> of</h2>' % node.rule
+ document.append('<h2>target is built using rule <tt>%s</tt> of</h2>' %
+ node.rule)
if len(node.inputs) > 0:
- print '<div class=filelist>'
+ document.append('<div class=filelist>')
for input, type in sorted(node.inputs):
extra = ''
if type:
extra = ' (%s)' % type
- print '<tt><a href="?%s">%s</a>%s</tt><br>' % (input, input, extra)
- print '</div>'
+ document.append('<tt><a href="?%s">%s</a>%s</tt><br>' %
+ (input, input, extra))
+ document.append('</div>')
if node.outputs:
- print '<h2>dependent edges build:</h2>'
- print '<div class=filelist>'
+ document.append('<h2>dependent edges build:</h2>')
+ document.append('<div class=filelist>')
for output in sorted(node.outputs):
- print '<tt><a href="?%s">%s</a></tt><br>' % (output, output)
- print '</div>'
+ document.append('<tt><a href="?%s">%s</a></tt><br>' %
+ (output, output))
+ document.append('</div>')
+
+ return '\n'.join(document)
def ninja_dump(target):
proc = subprocess.Popen([sys.argv[1], '-t', 'query', target],
- stdout=subprocess.PIPE)
- return proc.communicate()[0]
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True)
+ return proc.communicate() + (proc.returncode,)
-class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class RequestHandler(httpserver.BaseHTTPRequestHandler):
def do_GET(self):
assert self.path[0] == '/'
target = self.path[1:]
@@ -152,28 +165,28 @@ class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
return
target = target[1:]
- input = ninja_dump(target)
+ ninja_output, ninja_error, exit_code = ninja_dump(target)
+ if exit_code == 0:
+ page_body = generate_html(parse(ninja_output.strip()))
+ else:
+ # Relay ninja's error message.
+ page_body = '<h1><tt>%s</tt></h1>' % ninja_error
self.send_response(200)
self.end_headers()
- stdout = sys.stdout
- sys.stdout = self.wfile
- try:
- generate_html(parse(input.strip()))
- finally:
- sys.stdout = stdout
+ self.wfile.write(create_page(page_body).encode('utf-8'))
def log_message(self, format, *args):
pass # Swallow console spam.
port = 8000
-httpd = BaseHTTPServer.HTTPServer(('',port), RequestHandler)
+httpd = httpserver.HTTPServer(('',port), RequestHandler)
try:
- print 'Web server running on port %d, ctl-C to abort...' % port
+ print('Web server running on port %d, ctl-C to abort...' % port)
webbrowser.open_new('http://localhost:%s' % port)
httpd.serve_forever()
except KeyboardInterrupt:
- print
+ print()
pass # Swallow console spam.
diff --git a/src/build.cc b/src/build.cc
index e1aaad1..b4229c4 100644
--- a/src/build.cc
+++ b/src/build.cc
@@ -17,6 +17,7 @@
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
+#include <functional>
#ifdef _WIN32
#include <windows.h>
@@ -37,13 +38,50 @@
#include "subprocess.h"
#include "util.h"
+namespace {
+
+/// A CommandRunner that doesn't actually run the commands.
+struct DryRunCommandRunner : public CommandRunner {
+ virtual ~DryRunCommandRunner() {}
+
+ // Overridden from CommandRunner:
+ virtual bool CanRunMore();
+ virtual bool StartCommand(Edge* edge);
+ virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */);
+
+ private:
+ queue<Edge*> finished_;
+};
+
+bool DryRunCommandRunner::CanRunMore() {
+ return true;
+}
+
+bool DryRunCommandRunner::StartCommand(Edge* edge) {
+ finished_.push(edge);
+ return true;
+}
+
+Edge* DryRunCommandRunner::WaitForCommand(ExitStatus* status,
+ string* /*output*/) {
+ if (finished_.empty()) {
+ *status = ExitFailure;
+ return NULL;
+ }
+ *status = ExitSuccess;
+ Edge* edge = finished_.front();
+ finished_.pop();
+ return edge;
+}
+
+} // namespace
+
BuildStatus::BuildStatus(const BuildConfig& config)
: config_(config),
start_time_millis_(GetTimeMillis()),
started_edges_(0), finished_edges_(0), total_edges_(0),
have_blank_line_(true), progress_status_format_(NULL),
- overall_rate_(), current_rate_(),
- current_rate_average_count_(config.parallelism) {
+ overall_rate_(), current_rate_(config.parallelism) {
#ifndef _WIN32
const char* term = getenv("TERM");
smart_terminal_ = isatty(1) && term && string(term) != "dumb";
@@ -136,9 +174,11 @@ void BuildStatus::BuildFinished() {
printf("\n");
}
-string BuildStatus::FormatProgressStatus(const char* progress_status_format) const {
+string BuildStatus::FormatProgressStatus(
+ const char* progress_status_format) const {
string out;
char buf[32];
+ int percent;
for (const char* s = progress_status_format; *s != '\0'; ++s) {
if (*s == '%') {
++s;
@@ -177,30 +217,31 @@ string BuildStatus::FormatProgressStatus(const char* progress_status_format) con
out += buf;
break;
- // Overall finished edges per second.
+ // Overall finished edges per second.
case 'o':
- overall_rate_.UpdateRate(finished_edges_, finished_edges_);
- overall_rate_.snprinfRate(buf, "%.1f");
+ overall_rate_.UpdateRate(finished_edges_);
+ snprinfRate(overall_rate_.rate(), buf, "%.1f");
out += buf;
break;
- // Current rate, average over the last '-j' jobs.
+ // Current rate, average over the last '-j' jobs.
case 'c':
- // TODO use sliding window?
- if (finished_edges_ > current_rate_.last_update() &&
- finished_edges_ - current_rate_.last_update() == current_rate_average_count_) {
- current_rate_.UpdateRate(current_rate_average_count_, finished_edges_);
- current_rate_.Restart();
- }
- current_rate_.snprinfRate(buf, "%.0f");
+ current_rate_.UpdateRate(finished_edges_);
+ snprinfRate(current_rate_.rate(), buf, "%.1f");
+ out += buf;
+ break;
+
+ // Percentage
+ case 'p':
+ percent = (100 * started_edges_) / total_edges_;
+ snprintf(buf, sizeof(buf), "%3i%%", percent);
out += buf;
break;
- default: {
+ default:
Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s);
return "";
}
- }
} else {
out.push_back(*s);
}
@@ -325,7 +366,7 @@ bool Plan::AddSubTarget(Node* node, vector<Node*>* stack, string* err) {
want = true;
++wanted_edges_;
if (edge->AllInputsReady())
- ready_.insert(edge);
+ ScheduleWork(edge);
if (!edge->is_phony())
++command_edges_;
}
@@ -374,6 +415,22 @@ Edge* Plan::FindWork() {
return edge;
}
+void Plan::ScheduleWork(Edge* edge) {
+ Pool* pool = edge->pool();
+ if (pool->ShouldDelayEdge()) {
+ pool->DelayEdge(edge);
+ pool->RetrieveReadyEdges(&ready_);
+ } else {
+ pool->EdgeScheduled(*edge);
+ ready_.insert(edge);
+ }
+}
+
+void Plan::ResumeDelayedJobs(Edge* edge) {
+ edge->pool()->EdgeFinished(*edge);
+ edge->pool()->RetrieveReadyEdges(&ready_);
+}
+
void Plan::EdgeFinished(Edge* edge) {
map<Edge*, bool>::iterator i = want_.find(edge);
assert(i != want_.end());
@@ -382,6 +439,9 @@ void Plan::EdgeFinished(Edge* edge) {
want_.erase(i);
edge->outputs_ready_ = true;
+ // See if this job frees up any delayed jobs
+ ResumeDelayedJobs(edge);
+
// Check off any nodes we were waiting for with this edge.
for (vector<Node*>::iterator i = edge->outputs_.begin();
i != edge->outputs_.end(); ++i) {
@@ -400,7 +460,7 @@ void Plan::NodeFinished(Node* node) {
// See if the edge is now ready.
if ((*i)->AllInputsReady()) {
if (want_i->second) {
- ready_.insert(*i);
+ ScheduleWork(*i);
} else {
// We do not need to build this edge, but we might need to build one of
// its dependents.
@@ -422,8 +482,9 @@ void Plan::CleanNode(DependencyScan* scan, Node* node) {
// If all non-order-only inputs for this edge are now clean,
// we might have changed the dirty state of the outputs.
- vector<Node*>::iterator begin = (*ei)->inputs_.begin(),
- end = (*ei)->inputs_.end() - (*ei)->order_only_deps_;
+ vector<Node*>::iterator
+ begin = (*ei)->inputs_.begin(),
+ end = (*ei)->inputs_.end() - (*ei)->order_only_deps_;
if (find_if(begin, end, mem_fun(&Node::dirty)) == end) {
// Recompute most_recent_input and command.
Node* most_recent_input = NULL;
@@ -533,30 +594,6 @@ Edge* RealCommandRunner::WaitForCommand(ExitStatus* status, string* output) {
return edge;
}
-/// A CommandRunner that doesn't actually run the commands.
-struct DryRunCommandRunner : public CommandRunner {
- virtual ~DryRunCommandRunner() {}
- virtual bool CanRunMore() {
- return true;
- }
- virtual bool StartCommand(Edge* edge) {
- finished_.push(edge);
- return true;
- }
- virtual Edge* WaitForCommand(ExitStatus* status, string* /* output */) {
- if (finished_.empty()) {
- *status = ExitFailure;
- return NULL;
- }
- *status = ExitSuccess;
- Edge* edge = finished_.front();
- finished_.pop();
- return edge;
- }
-
- queue<Edge*> finished_;
-};
-
Builder::Builder(State* state, const BuildConfig& config,
BuildLog* log, DiskInterface* disk_interface)
: state_(state), config_(config), disk_interface_(disk_interface),
@@ -718,6 +755,7 @@ bool Builder::Build(string* err) {
}
bool Builder::StartEdge(Edge* edge, string* err) {
+ METRIC_RECORD("StartEdge");
if (edge->is_phony())
return true;
@@ -734,8 +772,10 @@ bool Builder::StartEdge(Edge* edge, string* err) {
// Create response file, if needed
// XXX: this may also block; do we care?
if (edge->HasRspFile()) {
- if (!disk_interface_->WriteFile(edge->GetRspFile(), edge->GetRspFileContent()))
+ if (!disk_interface_->WriteFile(edge->GetRspFile(),
+ edge->GetRspFileContent())) {
return false;
+ }
}
// start command computing and run it
@@ -748,6 +788,7 @@ bool Builder::StartEdge(Edge* edge, string* err) {
}
void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
+ METRIC_RECORD("FinishEdge");
TimeStamp restat_mtime = 0;
if (success) {
@@ -777,7 +818,8 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) {
}
if (restat_mtime != 0 && !edge->rule().depfile().empty()) {
- TimeStamp depfile_mtime = disk_interface_->Stat(edge->EvaluateDepFile());
+ TimeStamp depfile_mtime =
+ disk_interface_->Stat(edge->EvaluateDepFile());
if (depfile_mtime > restat_mtime)
restat_mtime = depfile_mtime;
}
diff --git a/src/build.h b/src/build.h
index 3e7a144..23f653e 100644
--- a/src/build.h
+++ b/src/build.h
@@ -15,13 +15,13 @@
#ifndef NINJA_BUILD_H_
#define NINJA_BUILD_H_
+#include <cstdio>
#include <map>
+#include <memory>
+#include <queue>
#include <set>
#include <string>
-#include <queue>
#include <vector>
-#include <memory>
-#include <cstdio>
#include "graph.h" // XXX needed for DependencyScan; should rearrange.
#include "exit_status.h"
@@ -70,6 +70,16 @@ private:
bool CheckDependencyCycle(Node* node, vector<Node*>* stack, string* err);
void NodeFinished(Node* node);
+ /// Submits a ready edge as a candidate for execution.
+ /// The edge may be delayed from running, for example if it's a member of a
+ /// currently-full pool.
+ void ScheduleWork(Edge* edge);
+
+ /// Allows jobs blocking on |edge| to potentially resume.
+ /// For example, if |edge| is a member of a pool, calling this may schedule
+ /// previously pending jobs in that pool.
+ void ResumeDelayedJobs(Edge* edge);
+
/// Keep track of which edges we want to build in this plan. If this map does
/// not contain an entry for an edge, we do not want to build the entry or its
/// dependents. If an entry maps to false, we do not want to build it, but we
@@ -175,7 +185,7 @@ struct BuildStatus {
/// Format the progress status string by replacing the placeholders.
/// See the user manual for more information about the available
/// placeholders.
- /// @param progress_status_format_ The format of the progress status.
+ /// @param progress_status_format The format of the progress status.
string FormatProgressStatus(const char* progress_status_format) const;
private:
@@ -200,38 +210,56 @@ struct BuildStatus {
/// The custom progress status format to use.
const char* progress_status_format_;
+ template<size_t S>
+ void snprinfRate(double rate, char(&buf)[S], const char* format) const {
+ if (rate == -1) snprintf(buf, S, "?");
+ else snprintf(buf, S, format, rate);
+ }
+
struct RateInfo {
- RateInfo() : last_update_(0), rate_(-1) {}
-
- double rate() const { return rate_; }
- int last_update() const { return last_update_; }
- void Restart() { return stopwatch_.Restart(); }
-
- double UpdateRate(int edges, int update_hint) {
- if (update_hint != last_update_) {
- rate_ = edges / stopwatch_.Elapsed() + 0.5;
- last_update_ = update_hint;
- }
- return rate_;
+ RateInfo() : rate_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int edges) {
+ if (edges && stopwatch_.Elapsed())
+ rate_ = edges / stopwatch_.Elapsed();
}
- template<class T>
- void snprinfRate(T buf, const char* format) {
- if (rate_ == -1)
- snprintf(buf, sizeof(buf), "?");
- else
- snprintf(buf, sizeof(buf), format, rate_);
+ private:
+ double rate_;
+ Stopwatch stopwatch_;
+ };
+
+ struct SlidingRateInfo {
+ SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {}
+
+ void Restart() { stopwatch_.Restart(); }
+ double rate() { return rate_; }
+
+ void UpdateRate(int update_hint) {
+ if (update_hint == last_update_)
+ return;
+ last_update_ = update_hint;
+
+ if (times_.size() == N)
+ times_.pop();
+ times_.push(stopwatch_.Elapsed());
+ if (times_.back() != times_.front())
+ rate_ = times_.size() / (times_.back() - times_.front());
}
private:
+ double rate_;
Stopwatch stopwatch_;
+ const size_t N;
+ std::queue<double> times_;
int last_update_;
- double rate_;
};
mutable RateInfo overall_rate_;
- mutable RateInfo current_rate_;
- const int current_rate_average_count_;
+ mutable SlidingRateInfo current_rate_;
#ifdef _WIN32
void* console_;
diff --git a/src/build_log.cc b/src/build_log.cc
index a633892..6b73002 100644
--- a/src/build_log.cc
+++ b/src/build_log.cc
@@ -56,7 +56,7 @@ uint64_t MurmurHash64A(const void* key, size_t len) {
uint64_t h = seed ^ (len * m);
const uint64_t * data = (const uint64_t *)key;
const uint64_t * end = data + (len/8);
- while(data != end) {
+ while (data != end) {
uint64_t k = *data++;
k *= m;
k ^= k >> r;
@@ -65,7 +65,7 @@ uint64_t MurmurHash64A(const void* key, size_t len) {
h *= m;
}
const unsigned char* data2 = (const unsigned char*)data;
- switch(len & 7)
+ switch (len & 7)
{
case 7: h ^= uint64_t(data2[6]) << 48;
case 6: h ^= uint64_t(data2[5]) << 40;
@@ -91,6 +91,15 @@ uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) {
return MurmurHash64A(command.str_, command.len_);
}
+BuildLog::LogEntry::LogEntry(const string& output)
+ : output(output) {}
+
+BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash,
+ int start_time, int end_time, TimeStamp restat_mtime)
+ : output(output), command_hash(command_hash),
+ start_time(start_time), end_time(end_time), restat_mtime(restat_mtime)
+{}
+
BuildLog::BuildLog()
: log_file_(NULL), needs_recompaction_(false) {}
@@ -130,6 +139,7 @@ bool BuildLog::OpenForWrite(const string& path, string* err) {
void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
TimeStamp restat_mtime) {
string command = edge->EvaluateCommand(true);
+ uint64_t command_hash = LogEntry::HashCommand(command);
for (vector<Node*>::iterator out = edge->outputs_.begin();
out != edge->outputs_.end(); ++out) {
const string& path = (*out)->path();
@@ -138,11 +148,10 @@ void BuildLog::RecordCommand(Edge* edge, int start_time, int end_time,
if (i != entries_.end()) {
log_entry = i->second;
} else {
- log_entry = new LogEntry;
- log_entry->output = path;
+ log_entry = new LogEntry(path);
entries_.insert(Entries::value_type(log_entry->output, log_entry));
}
- log_entry->command_hash = LogEntry::HashCommand(command);
+ log_entry->command_hash = command_hash;
log_entry->start_time = start_time;
log_entry->end_time = end_time;
log_entry->restat_mtime = restat_mtime;
@@ -158,8 +167,7 @@ void BuildLog::Close() {
log_file_ = NULL;
}
-class LineReader {
- public:
+struct LineReader {
explicit LineReader(FILE* file)
: file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) {
memset(buf_, 0, sizeof(buf_));
@@ -287,8 +295,7 @@ bool BuildLog::Load(const string& path, string* err) {
if (i != entries_.end()) {
entry = i->second;
} else {
- entry = new LogEntry;
- entry->output = output;
+ entry = new LogEntry(output);
entries_.insert(Entries::value_type(entry->output, entry));
++unique_entry_count;
}
@@ -341,6 +348,7 @@ void BuildLog::WriteEntry(FILE* f, const LogEntry& entry) {
}
bool BuildLog::Recompact(const string& path, string* err) {
+ METRIC_RECORD(".ninja_log recompact");
printf("Recompacting log...\n");
string temp_path = path + ".recompact";
diff --git a/src/build_log.h b/src/build_log.h
index 4141ff3..231bfd9 100644
--- a/src/build_log.h
+++ b/src/build_log.h
@@ -60,6 +60,10 @@ struct BuildLog {
start_time == o.start_time && end_time == o.end_time &&
restat_mtime == o.restat_mtime;
}
+
+ explicit LogEntry(const string& output);
+ LogEntry(const string& output, uint64_t command_hash,
+ int start_time, int end_time, TimeStamp restat_mtime);
};
/// Lookup a previously-run command by its output path.
diff --git a/src/build_log_test.cc b/src/build_log_test.cc
index a6c2a86..2dd6500 100644
--- a/src/build_log_test.cc
+++ b/src/build_log_test.cc
@@ -26,6 +26,8 @@
#include <unistd.h>
#endif
+namespace {
+
const char kTestFilename[] = "BuildLogTest-tempfile";
struct BuildLogTest : public StateTestWithBuiltinRules {
@@ -145,7 +147,8 @@ TEST_F(BuildLogTest, Truncate) {
ASSERT_EQ(0, truncate(kTestFilename, size));
#else
int fh;
- fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO, _S_IREAD | _S_IWRITE);
+ fh = _sopen(kTestFilename, _O_RDWR | _O_CREAT, _SH_DENYNO,
+ _S_IREAD | _S_IWRITE);
ASSERT_EQ(0, _chsize(fh, size));
_close(fh);
#endif
@@ -245,3 +248,25 @@ TEST_F(BuildLogTest, VeryLongInputLine) {
ASSERT_EQ(789, e->restat_mtime);
ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash));
}
+
+TEST_F(BuildLogTest, MultiTargetEdge) {
+ AssertParse(&state_,
+"build out out.d: cat\n");
+
+ BuildLog log;
+ log.RecordCommand(state_.edges_[0], 21, 22);
+
+ ASSERT_EQ(2u, log.entries().size());
+ BuildLog::LogEntry* e1 = log.LookupByOutput("out");
+ ASSERT_TRUE(e1);
+ BuildLog::LogEntry* e2 = log.LookupByOutput("out.d");
+ ASSERT_TRUE(e2);
+ ASSERT_EQ("out", e1->output);
+ ASSERT_EQ("out.d", e2->output);
+ ASSERT_EQ(21, e1->start_time);
+ ASSERT_EQ(21, e2->start_time);
+ ASSERT_EQ(22, e2->end_time);
+ ASSERT_EQ(22, e2->end_time);
+}
+
+} // anonymous namespace
diff --git a/src/build_test.cc b/src/build_test.cc
index 859e758..59c4c53 100644
--- a/src/build_test.cc
+++ b/src/build_test.cc
@@ -176,6 +176,132 @@ TEST_F(PlanTest, DependencyCycle) {
ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err);
}
+TEST_F(PlanTest, PoolWithDepthOne) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool foobar\n"
+" depth = 1\n"
+"rule poolcat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"build out1: poolcat in\n"
+"build out2: poolcat in\n"));
+ GetNode("out1")->MarkDirty();
+ GetNode("out2")->MarkDirty();
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out1"), &err));
+ ASSERT_EQ("", err);
+ EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err));
+ ASSERT_EQ("", err);
+ ASSERT_TRUE(plan_.more_to_do());
+
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out1", edge->outputs_[0]->path());
+
+ // This will be false since poolcat is serialized
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge);
+
+ edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("out2", edge->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(edge);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ edge = plan_.FindWork();
+ ASSERT_EQ(0, edge);
+}
+
+TEST_F(PlanTest, PoolsWithDepthTwo) {
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+"pool foobar\n"
+" depth = 2\n"
+"pool bazbin\n"
+" depth = 2\n"
+"rule foocat\n"
+" command = cat $in > $out\n"
+" pool = foobar\n"
+"rule bazcat\n"
+" command = cat $in > $out\n"
+" pool = bazbin\n"
+"build out1: foocat in\n"
+"build out2: foocat in\n"
+"build out3: foocat in\n"
+"build outb1: bazcat in\n"
+"build outb2: bazcat in\n"
+"build outb3: bazcat in\n"
+" pool =\n"
+"build allTheThings: cat out1 out2 out3 outb1 outb2 outb3\n"
+));
+ // Mark all the out* nodes dirty
+ for (int i = 0; i < 3; ++i) {
+ GetNode("out" + string(1, '1' + i))->MarkDirty();
+ GetNode("outb" + string(1, '1' + i))->MarkDirty();
+ }
+ GetNode("allTheThings")->MarkDirty();
+
+ string err;
+ EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err));
+ ASSERT_EQ("", err);
+
+ // Grab the first 4 edges, out1 out2 outb1 outb2
+ deque<Edge*> edges;
+ for (int i = 0; i < 4; ++i) {
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ string base_name(i < 2 ? "out" : "outb");
+ ASSERT_EQ(base_name + string(1, '1' + (i % 2)), edge->outputs_[0]->path());
+ edges.push_back(edge);
+ }
+
+ // outb3 is exempt because it has an empty pool
+ ASSERT_TRUE(plan_.more_to_do());
+ Edge* edge = plan_.FindWork();
+ ASSERT_TRUE(edge);
+ ASSERT_EQ("in", edge->inputs_[0]->path());
+ ASSERT_EQ("outb3", edge->outputs_[0]->path());
+ edges.push_back(edge);
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ // finish out1
+ plan_.EdgeFinished(edges.front());
+ edges.pop_front();
+
+ // out3 should be available
+ Edge* out3 = plan_.FindWork();
+ ASSERT_TRUE(out3);
+ ASSERT_EQ("in", out3->inputs_[0]->path());
+ ASSERT_EQ("out3", out3->outputs_[0]->path());
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ plan_.EdgeFinished(out3);
+
+ ASSERT_FALSE(plan_.FindWork());
+
+ for (deque<Edge*>::iterator it = edges.begin(); it != edges.end(); ++it) {
+ plan_.EdgeFinished(*it);
+ }
+
+ Edge* final = plan_.FindWork();
+ ASSERT_TRUE(final);
+ ASSERT_EQ("allTheThings", final->outputs_[0]->path());
+
+ plan_.EdgeFinished(final);
+
+ ASSERT_FALSE(plan_.more_to_do());
+ ASSERT_FALSE(plan_.FindWork());
+}
+
struct BuildTest : public StateTestWithBuiltinRules,
public CommandRunner {
BuildTest() : config_(MakeConfig()),
@@ -447,10 +573,12 @@ TEST_F(BuildTest, MakeDirs) {
string err;
#ifdef _WIN32
- ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir\\dir2\\file: cat in1\n"));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir\\dir2\\file: cat in1\n"));
EXPECT_TRUE(builder_.AddTarget("subdir\\dir2\\file", &err));
#else
- ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build subdir/dir2/file: cat in1\n"));
+ ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+ "build subdir/dir2/file: cat in1\n"));
EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err));
#endif
@@ -869,7 +997,7 @@ TEST_F(BuildWithLogTest, RestatMissingInput) {
// Create all necessary files
fs_.Create("in", now_, "");
- // The implicit dependencies and the depfile itself
+ // The implicit dependencies and the depfile itself
// are newer than the output
TimeStamp restat_mtime = ++now_;
fs_.Create("out1.d", now_, "out1: will.be.deleted restat.file\n");
@@ -889,10 +1017,10 @@ TEST_F(BuildWithLogTest, RestatMissingInput) {
ASSERT_TRUE(NULL != log_entry);
ASSERT_EQ(restat_mtime, log_entry->restat_mtime);
- // Now remove a file, referenced from depfile, so that target becomes
+ // Now remove a file, referenced from depfile, so that target becomes
// dirty, but the output does not change
fs_.RemoveFile("will.be.deleted");
-
+
// Trigger the build again - only out1 gets built
commands_ran_.clear();
state_.Reset();
@@ -943,7 +1071,7 @@ TEST_F(BuildDryRun, AllCommandsShown) {
}
// Test that RSP files are created when & where appropriate and deleted after
-// succesful execution.
+// successful execution.
TEST_F(BuildTest, RspFileSuccess)
{
ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
@@ -1135,3 +1263,4 @@ TEST_F(BuildTest, StatusFormatReplacePlaceholder) {
EXPECT_EQ("[%/s0/t0/r0/u0/f0]",
status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]"));
}
+
diff --git a/src/clean.cc b/src/clean.cc
index 3fe23ec..0b8476b 100644
--- a/src/clean.cc
+++ b/src/clean.cc
@@ -29,6 +29,7 @@ Cleaner::Cleaner(State* state, const BuildConfig& config)
: state_(state),
config_(config),
removed_(),
+ cleaned_(),
cleaned_files_count_(0),
disk_interface_(new RealDiskInterface),
status_(0) {
@@ -40,6 +41,7 @@ Cleaner::Cleaner(State* state,
: state_(state),
config_(config),
removed_(),
+ cleaned_(),
cleaned_files_count_(0),
disk_interface_(disk_interface),
status_(0) {
@@ -80,6 +82,16 @@ bool Cleaner::IsAlreadyRemoved(const string& path) {
return (i != removed_.end());
}
+void Cleaner::RemoveEdgeFiles(Edge* edge) {
+ string depfile = edge->EvaluateDepFile();
+ if (!depfile.empty())
+ Remove(depfile);
+
+ string rspfile = edge->GetRspFile();
+ if (!rspfile.empty())
+ Remove(rspfile);
+}
+
void Cleaner::PrintHeader() {
if (config_.verbosity == BuildConfig::QUIET)
return;
@@ -111,12 +123,8 @@ int Cleaner::CleanAll(bool generator) {
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
}
- // Remove the depfile
- if (!(*e)->rule().depfile().empty())
- Remove((*e)->EvaluateDepFile());
- // Remove the response file
- if ((*e)->HasRspFile())
- Remove((*e)->GetRspFile());
+
+ RemoveEdgeFiles(*e);
}
PrintFooter();
return status_;
@@ -127,16 +135,20 @@ void Cleaner::DoCleanTarget(Node* target) {
// Do not try to remove phony targets
if (!e->is_phony()) {
Remove(target->path());
- if (!target->in_edge()->rule().depfile().empty())
- Remove(target->in_edge()->EvaluateDepFile());
- if (e->HasRspFile())
- Remove(e->GetRspFile());
+ RemoveEdgeFiles(e);
}
for (vector<Node*>::iterator n = e->inputs_.begin(); n != e->inputs_.end();
++n) {
- DoCleanTarget(*n);
+ Node* next = *n;
+ // call DoCleanTarget recursively if this node has not been visited
+ if (cleaned_.count(next) == 0) {
+ DoCleanTarget(next);
+ }
}
}
+
+ // mark this target to be cleaned already
+ cleaned_.insert(target);
}
int Cleaner::CleanTarget(Node* target) {
@@ -191,10 +203,7 @@ void Cleaner::DoCleanRule(const Rule* rule) {
for (vector<Node*>::iterator out_node = (*e)->outputs_.begin();
out_node != (*e)->outputs_.end(); ++out_node) {
Remove((*out_node)->path());
- if (!(*e)->rule().depfile().empty())
- Remove((*e)->EvaluateDepFile());
- if ((*e)->HasRspFile())
- Remove((*e)->GetRspFile());
+ RemoveEdgeFiles(*e);
}
}
}
@@ -249,4 +258,5 @@ void Cleaner::Reset() {
status_ = 0;
cleaned_files_count_ = 0;
removed_.clear();
+ cleaned_.clear();
}
diff --git a/src/clean.h b/src/clean.h
index 5938dff..19432ab 100644
--- a/src/clean.h
+++ b/src/clean.h
@@ -27,8 +27,7 @@ struct Node;
struct Rule;
struct DiskInterface;
-class Cleaner {
- public:
+struct Cleaner {
/// Build a cleaner object with a real disk interface.
Cleaner(State* state, const BuildConfig& config);
@@ -81,10 +80,14 @@ class Cleaner {
/// @returns whether the file @a path exists.
bool FileExists(const string& path);
void Report(const string& path);
+
/// Remove the given @a path file only if it has not been already removed.
void Remove(const string& path);
/// @return whether the given @a path has already been removed.
bool IsAlreadyRemoved(const string& path);
+ /// Remove the depfile and rspfile for an Edge.
+ void RemoveEdgeFiles(Edge* edge);
+
/// Helper recursive method for CleanTarget().
void DoCleanTarget(Node* target);
void PrintHeader();
@@ -95,6 +98,7 @@ class Cleaner {
State* state_;
const BuildConfig& config_;
set<string> removed_;
+ set<Node*> cleaned_;
int cleaned_files_count_;
DiskInterface* disk_interface_;
int status_;
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 515ff59..7c557cd 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -80,7 +80,8 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
// MSDN: "Naming Files, Paths, and Namespaces"
// http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) {
- Error("Stat(%s): Filename longer than %i characters", path.c_str(), MAX_PATH);
+ Error("Stat(%s): Filename longer than %i characters",
+ path.c_str(), MAX_PATH);
return -1;
}
WIN32_FILE_ATTRIBUTE_DATA attrs;
@@ -116,18 +117,21 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
bool RealDiskInterface::WriteFile(const string& path, const string& contents) {
FILE * fp = fopen(path.c_str(), "w");
if (fp == NULL) {
- Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to create file. %s",
+ path.c_str(), strerror(errno));
return false;
}
if (fwrite(contents.data(), 1, contents.length(), fp) < contents.length()) {
- Error("WriteFile(%s): Unable to write to the file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to write to the file. %s",
+ path.c_str(), strerror(errno));
fclose(fp);
return false;
}
if (fclose(fp) == EOF) {
- Error("WriteFile(%s): Unable to close the file. %s", path.c_str(), strerror(errno));
+ Error("WriteFile(%s): Unable to close the file. %s",
+ path.c_str(), strerror(errno));
return false;
}
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index 32fe9cb..c2315c7 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -25,8 +25,7 @@
namespace {
-class DiskInterfaceTest : public testing::Test {
- public:
+struct DiskInterfaceTest : public testing::Test {
virtual void SetUp() {
// These tests do real disk accesses, so create a temp dir.
temp_dir_.CreateAndEnter("Ninja-DiskInterfaceTest");
diff --git a/src/graph.cc b/src/graph.cc
index 6ae324d..f9b9c6f 100644
--- a/src/graph.cc
+++ b/src/graph.cc
@@ -145,9 +145,10 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge,
if (edge->rule_->restat() && build_log() &&
(entry = build_log()->LookupByOutput(output->path()))) {
if (entry->restat_mtime < most_recent_stamp) {
- EXPLAIN("restat of output %s older than most recent input %s (%d vs %d)",
- output->path().c_str(), most_recent_input->path().c_str(),
- entry->restat_mtime, most_recent_stamp);
+ EXPLAIN("restat of output %s older than most recent input %s "
+ "(%d vs %d)",
+ output->path().c_str(), most_recent_input->path().c_str(),
+ entry->restat_mtime, most_recent_stamp);
return true;
}
} else {
@@ -192,7 +193,7 @@ struct EdgeEnv : public Env {
virtual string LookupVariable(const string& var);
/// Given a span of Nodes, construct a list of paths suitable for a command
- /// line. XXX here is where shell-escaping of e.g spaces should happen.
+ /// line.
string MakePathList(vector<Node*>::iterator begin,
vector<Node*>::iterator end,
char sep);
@@ -214,7 +215,6 @@ string EdgeEnv::LookupVariable(const string& var) {
} else if (edge_->env_) {
return edge_->env_->LookupVariable(var);
} else {
- // XXX should we warn here?
return string();
}
}
@@ -241,7 +241,7 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin,
string Edge::EvaluateCommand(bool incl_rsp_file) {
EdgeEnv env(this);
string command = rule_->command().Evaluate(&env);
- if (incl_rsp_file && HasRspFile())
+ if (incl_rsp_file && HasRspFile())
command += ";rspfile=" + GetRspFileContent();
return command;
}
@@ -317,7 +317,8 @@ bool DependencyScan::LoadDepFile(Edge* edge, string* err) {
// create one; this makes us not abort if the input is missing,
// but instead will rebuild in that circumstance.
if (!node->in_edge()) {
- Edge* phony_edge = state_->AddEdge(&State::kPhonyRule);
+ Edge* phony_edge = state_->AddEdge(&State::kPhonyRule,
+ &State::kDefaultPool);
node->set_in_edge(phony_edge);
phony_edge->outputs_.push_back(node);
@@ -345,6 +346,13 @@ void Edge::Dump(const char* prefix) const {
i != outputs_.end() && *i != NULL; ++i) {
printf("%s ", (*i)->path().c_str());
}
+ if (pool_) {
+ if (!pool_->name().empty()) {
+ printf("(in pool '%s')", pool_->name().c_str());
+ }
+ } else {
+ printf("(null pool?)");
+ }
printf("] 0x%p\n", this);
}
@@ -353,18 +361,18 @@ bool Edge::is_phony() const {
}
void Node::Dump(const char* prefix) const {
- printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
- prefix, path().c_str(), this,
- mtime(), mtime()?"":" (:missing)",
- dirty()?" dirty":" clean");
- if (in_edge()) {
- in_edge()->Dump("in-edge: ");
- }else{
- printf("no in-edge\n");
- }
- printf(" out edges:\n");
- for (vector<Edge*>::const_iterator e = out_edges().begin();
- e != out_edges().end() && *e != NULL; ++e) {
- (*e)->Dump(" +- ");
- }
+ printf("%s <%s 0x%p> mtime: %d%s, (:%s), ",
+ prefix, path().c_str(), this,
+ mtime(), mtime() ? "" : " (:missing)",
+ dirty() ? " dirty" : " clean");
+ if (in_edge()) {
+ in_edge()->Dump("in-edge: ");
+ } else {
+ printf("no in-edge\n");
+ }
+ printf(" out edges:\n");
+ for (vector<Edge*>::const_iterator e = out_edges().begin();
+ e != out_edges().end() && *e != NULL; ++e) {
+ (*e)->Dump(" +- ");
+ }
}
diff --git a/src/graph.h b/src/graph.h
index 272fcb9..3c31e19 100644
--- a/src/graph.h
+++ b/src/graph.h
@@ -131,6 +131,7 @@ struct Rule {
EvalString command_;
EvalString description_;
EvalString depfile_;
+ EvalString pool_;
EvalString rspfile_;
EvalString rspfile_content_;
};
@@ -138,6 +139,7 @@ struct Rule {
struct BuildLog;
struct Node;
struct State;
+struct Pool;
/// An edge in the dependency graph; links between Nodes using Rules.
struct Edge {
@@ -150,7 +152,7 @@ struct Edge {
/// Expand all variables in a command and return it as a string.
/// If incl_rsp_file is enabled, the string will also contain the
/// full contents of a response file (if applicable)
- string EvaluateCommand(bool incl_rsp_file = false); // XXX move to env, take env ptr
+ string EvaluateCommand(bool incl_rsp_file = false);
string EvaluateDepFile();
string GetDescription();
@@ -166,25 +168,25 @@ struct Edge {
void Dump(const char* prefix="") const;
const Rule* rule_;
+ Pool* pool_;
vector<Node*> inputs_;
vector<Node*> outputs_;
Env* env_;
bool outputs_ready_;
const Rule& rule() const { return *rule_; }
+ Pool* pool() const { return pool_; }
+ int weight() const { return 1; }
bool outputs_ready() const { return outputs_ready_; }
- // XXX There are three types of inputs.
+ // There are three types of inputs.
// 1) explicit deps, which show up as $in on the command line;
// 2) implicit deps, which the target depends on implicitly (e.g. C headers),
// and changes in them cause the target to rebuild;
// 3) order-only deps, which are needed before the target builds but which
// don't cause the target to rebuild.
- // Currently we stuff all of these into inputs_ and keep counts of #2 and #3
- // when we need to compute subsets. This is suboptimal; should think of a
- // better representation. (Could make each pointer into a pair of a pointer
- // and a type of input, or if memory matters could use the low bits of the
- // pointer...)
+ // These are stored in inputs_ in that order, and we keep counts of
+ // #2 and #3 when we need to access the various subsets.
int implicit_deps_;
int order_only_deps_;
bool is_implicit(size_t index) {
diff --git a/src/hash_map.h b/src/hash_map.h
index 9904fb8..076f6c0 100644
--- a/src/hash_map.h
+++ b/src/hash_map.h
@@ -25,7 +25,7 @@ unsigned int MurmurHash2(const void* key, size_t len) {
const int r = 24;
unsigned int h = seed ^ len;
const unsigned char * data = (const unsigned char *)key;
- while(len >= 4) {
+ while (len >= 4) {
unsigned int k = *(unsigned int *)data;
k *= m;
k ^= k >> r;
@@ -35,7 +35,7 @@ unsigned int MurmurHash2(const void* key, size_t len) {
data += 4;
len -= 4;
}
- switch(len) {
+ switch (len) {
case 3: h ^= data[2] << 16;
case 2: h ^= data[1] << 8;
case 1: h ^= data[0];
diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc
index 77b5b3b..29e6755 100644
--- a/src/includes_normalize_test.cc
+++ b/src/includes_normalize_test.cc
@@ -40,7 +40,8 @@ string GetCurDir() {
TEST(IncludesNormalize, WithRelative) {
string currentdir = IncludesNormalize::ToLower(GetCurDir());
EXPECT_EQ("c", IncludesNormalize::Normalize("a/b/c", "a/b"));
- EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"), NULL));
+ EXPECT_EQ("a", IncludesNormalize::Normalize(IncludesNormalize::AbsPath("a"),
+ NULL));
EXPECT_EQ(string("..\\") + currentdir + string("\\a"),
IncludesNormalize::Normalize("a", "../b"));
EXPECT_EQ(string("..\\") + currentdir + string("\\a\\b"),
@@ -69,16 +70,21 @@ TEST(IncludesNormalize, Join) {
}
TEST(IncludesNormalize, Split) {
- EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'), ':'));
- EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'), ':'));
- EXPECT_EQ("a:b:c", IncludesNormalize::Join(IncludesNormalize::Split("a/b/c", '/'), ':'));
+ EXPECT_EQ("", IncludesNormalize::Join(IncludesNormalize::Split("", '/'),
+ ':'));
+ EXPECT_EQ("a", IncludesNormalize::Join(IncludesNormalize::Split("a", '/'),
+ ':'));
+ EXPECT_EQ("a:b:c",
+ IncludesNormalize::Join(
+ IncludesNormalize::Split("a/b/c", '/'), ':'));
}
TEST(IncludesNormalize, ToLower) {
EXPECT_EQ("", IncludesNormalize::ToLower(""));
EXPECT_EQ("stuff", IncludesNormalize::ToLower("Stuff"));
EXPECT_EQ("stuff and things", IncludesNormalize::ToLower("Stuff AND thINGS"));
- EXPECT_EQ("stuff 3and thin43gs", IncludesNormalize::ToLower("Stuff 3AND thIN43GS"));
+ EXPECT_EQ("stuff 3and thin43gs",
+ IncludesNormalize::ToLower("Stuff 3AND thIN43GS"));
}
TEST(IncludesNormalize, DifferentDrive) {
diff --git a/src/lexer.cc b/src/lexer.cc
index 5d7d185..685fe81 100644
--- a/src/lexer.cc
+++ b/src/lexer.cc
@@ -83,6 +83,7 @@ const char* Lexer::TokenName(Token t) {
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
case TEOF: return "eof";
@@ -162,63 +163,71 @@ Lexer::Token Lexer::ReadToken() {
};
yych = *p;
- if (yych <= 'Z') {
+ if (yych <= '^') {
if (yych <= ',') {
if (yych <= 0x1F) {
- if (yych <= 0x00) goto yy21;
+ if (yych <= 0x00) goto yy22;
if (yych == '\n') goto yy6;
- goto yy23;
+ goto yy24;
} else {
if (yych <= ' ') goto yy2;
if (yych == '#') goto yy4;
- goto yy23;
+ goto yy24;
}
} else {
if (yych <= ':') {
- if (yych == '/') goto yy23;
- if (yych <= '9') goto yy20;
- goto yy14;
+ if (yych == '/') goto yy24;
+ if (yych <= '9') goto yy21;
+ goto yy15;
} else {
- if (yych == '=') goto yy12;
- if (yych <= '@') goto yy23;
- goto yy20;
+ if (yych <= '=') {
+ if (yych <= '<') goto yy24;
+ goto yy13;
+ } else {
+ if (yych <= '@') goto yy24;
+ if (yych <= 'Z') goto yy21;
+ goto yy24;
+ }
}
}
} else {
- if (yych <= 'h') {
- if (yych <= 'a') {
- if (yych == '_') goto yy20;
- if (yych <= '`') goto yy23;
- goto yy20;
+ if (yych <= 'i') {
+ if (yych <= 'b') {
+ if (yych == '`') goto yy24;
+ if (yych <= 'a') goto yy21;
+ goto yy8;
} else {
- if (yych <= 'b') goto yy8;
- if (yych == 'd') goto yy11;
- goto yy20;
+ if (yych == 'd') goto yy12;
+ if (yych <= 'h') goto yy21;
+ goto yy19;
}
} else {
- if (yych <= 's') {
- if (yych <= 'i') goto yy18;
- if (yych <= 'q') goto yy20;
- if (yych <= 'r') goto yy10;
- goto yy19;
+ if (yych <= 'r') {
+ if (yych == 'p') goto yy10;
+ if (yych <= 'q') goto yy21;
+ goto yy11;
} else {
- if (yych <= 'z') goto yy20;
- if (yych == '|') goto yy16;
- goto yy23;
+ if (yych <= 'z') {
+ if (yych <= 's') goto yy20;
+ goto yy21;
+ } else {
+ if (yych == '|') goto yy17;
+ goto yy24;
+ }
}
}
}
yy2:
yyaccept = 0;
yych = *(q = ++p);
- goto yy65;
+ goto yy70;
yy3:
{ token = INDENT; break; }
yy4:
yyaccept = 1;
yych = *(q = ++p);
if (yych <= 0x00) goto yy5;
- if (yych != '\r') goto yy60;
+ if (yych != '\r') goto yy65;
yy5:
{ token = ERROR; break; }
yy6:
@@ -227,159 +236,173 @@ yy7:
{ token = NEWLINE; break; }
yy8:
++p;
- if ((yych = *p) == 'u') goto yy54;
- goto yy25;
+ if ((yych = *p) == 'u') goto yy59;
+ goto yy26;
yy9:
{ token = IDENT; break; }
yy10:
yych = *++p;
- if (yych == 'u') goto yy50;
- goto yy25;
+ if (yych == 'o') goto yy55;
+ goto yy26;
yy11:
yych = *++p;
- if (yych == 'e') goto yy43;
- goto yy25;
+ if (yych == 'u') goto yy51;
+ goto yy26;
yy12:
+ yych = *++p;
+ if (yych == 'e') goto yy44;
+ goto yy26;
+yy13:
++p;
{ token = EQUALS; break; }
-yy14:
+yy15:
++p;
{ token = COLON; break; }
-yy16:
+yy17:
++p;
- if ((yych = *p) == '|') goto yy41;
+ if ((yych = *p) == '|') goto yy42;
{ token = PIPE; break; }
-yy18:
- yych = *++p;
- if (yych == 'n') goto yy34;
- goto yy25;
yy19:
yych = *++p;
- if (yych == 'u') goto yy26;
- goto yy25;
+ if (yych == 'n') goto yy35;
+ goto yy26;
yy20:
yych = *++p;
- goto yy25;
+ if (yych == 'u') goto yy27;
+ goto yy26;
yy21:
+ yych = *++p;
+ goto yy26;
+yy22:
++p;
{ token = TEOF; break; }
-yy23:
+yy24:
yych = *++p;
goto yy5;
-yy24:
+yy25:
++p;
yych = *p;
-yy25:
+yy26:
if (yybm[0+yych] & 32) {
- goto yy24;
+ goto yy25;
}
goto yy9;
-yy26:
+yy27:
yych = *++p;
- if (yych != 'b') goto yy25;
+ if (yych != 'b') goto yy26;
yych = *++p;
- if (yych != 'n') goto yy25;
+ if (yych != 'n') goto yy26;
yych = *++p;
- if (yych != 'i') goto yy25;
+ if (yych != 'i') goto yy26;
yych = *++p;
- if (yych != 'n') goto yy25;
+ if (yych != 'n') goto yy26;
yych = *++p;
- if (yych != 'j') goto yy25;
+ if (yych != 'j') goto yy26;
yych = *++p;
- if (yych != 'a') goto yy25;
+ if (yych != 'a') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = SUBNINJA; break; }
-yy34:
+yy35:
yych = *++p;
- if (yych != 'c') goto yy25;
+ if (yych != 'c') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 'u') goto yy25;
+ if (yych != 'u') goto yy26;
yych = *++p;
- if (yych != 'd') goto yy25;
+ if (yych != 'd') goto yy26;
yych = *++p;
- if (yych != 'e') goto yy25;
+ if (yych != 'e') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = INCLUDE; break; }
-yy41:
+yy42:
++p;
{ token = PIPE2; break; }
-yy43:
+yy44:
yych = *++p;
- if (yych != 'f') goto yy25;
+ if (yych != 'f') goto yy26;
yych = *++p;
- if (yych != 'a') goto yy25;
+ if (yych != 'a') goto yy26;
yych = *++p;
- if (yych != 'u') goto yy25;
+ if (yych != 'u') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 't') goto yy25;
+ if (yych != 't') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = DEFAULT; break; }
-yy50:
+yy51:
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
yych = *++p;
- if (yych != 'e') goto yy25;
+ if (yych != 'e') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = RULE; break; }
-yy54:
+yy55:
yych = *++p;
- if (yych != 'i') goto yy25;
+ if (yych != 'o') goto yy26;
yych = *++p;
- if (yych != 'l') goto yy25;
+ if (yych != 'l') goto yy26;
+ ++p;
+ if (yybm[0+(yych = *p)] & 32) {
+ goto yy25;
+ }
+ { token = POOL; break; }
+yy59:
+ yych = *++p;
+ if (yych != 'i') goto yy26;
yych = *++p;
- if (yych != 'd') goto yy25;
+ if (yych != 'l') goto yy26;
+ yych = *++p;
+ if (yych != 'd') goto yy26;
++p;
if (yybm[0+(yych = *p)] & 32) {
- goto yy24;
+ goto yy25;
}
{ token = BUILD; break; }
-yy59:
+yy64:
++p;
yych = *p;
-yy60:
+yy65:
if (yybm[0+yych] & 64) {
- goto yy59;
+ goto yy64;
}
- if (yych <= 0x00) goto yy61;
- if (yych <= '\f') goto yy62;
-yy61:
+ if (yych <= 0x00) goto yy66;
+ if (yych <= '\f') goto yy67;
+yy66:
p = q;
if (yyaccept <= 0) {
goto yy3;
} else {
goto yy5;
}
-yy62:
+yy67:
++p;
{ continue; }
-yy64:
+yy69:
yyaccept = 0;
q = ++p;
yych = *p;
-yy65:
+yy70:
if (yybm[0+yych] & 128) {
- goto yy64;
+ goto yy69;
}
- if (yych == '\n') goto yy66;
- if (yych == '#') goto yy59;
+ if (yych == '\n') goto yy71;
+ if (yych == '#') goto yy64;
goto yy3;
-yy66:
+yy71:
++p;
yych = *p;
goto yy7;
@@ -445,39 +468,39 @@ void Lexer::EatWhitespace() {
};
yych = *p;
if (yych <= ' ') {
- if (yych <= 0x00) goto yy73;
- if (yych <= 0x1F) goto yy75;
+ if (yych <= 0x00) goto yy78;
+ if (yych <= 0x1F) goto yy80;
} else {
- if (yych == '$') goto yy71;
- goto yy75;
+ if (yych == '$') goto yy76;
+ goto yy80;
}
++p;
yych = *p;
- goto yy79;
-yy70:
+ goto yy84;
+yy75:
{ continue; }
-yy71:
+yy76:
++p;
- if ((yych = *p) == '\n') goto yy76;
-yy72:
+ if ((yych = *p) == '\n') goto yy81;
+yy77:
{ break; }
-yy73:
+yy78:
++p;
{ break; }
-yy75:
+yy80:
yych = *++p;
- goto yy72;
-yy76:
+ goto yy77;
+yy81:
++p;
{ continue; }
-yy78:
+yy83:
++p;
yych = *p;
-yy79:
+yy84:
if (yybm[0+yych] & 128) {
- goto yy78;
+ goto yy83;
}
- goto yy70;
+ goto yy75;
}
}
@@ -527,40 +550,40 @@ bool Lexer::ReadIdent(string* out) {
yych = *p;
if (yych <= '@') {
if (yych <= '.') {
- if (yych <= ',') goto yy84;
+ if (yych <= ',') goto yy89;
} else {
- if (yych <= '/') goto yy84;
- if (yych >= ':') goto yy84;
+ if (yych <= '/') goto yy89;
+ if (yych >= ':') goto yy89;
}
} else {
if (yych <= '_') {
- if (yych <= 'Z') goto yy82;
- if (yych <= '^') goto yy84;
+ if (yych <= 'Z') goto yy87;
+ if (yych <= '^') goto yy89;
} else {
- if (yych <= '`') goto yy84;
- if (yych >= '{') goto yy84;
+ if (yych <= '`') goto yy89;
+ if (yych >= '{') goto yy89;
}
}
-yy82:
+yy87:
++p;
yych = *p;
- goto yy87;
-yy83:
+ goto yy92;
+yy88:
{
out->assign(start, p - start);
break;
}
-yy84:
+yy89:
++p;
{ return false; }
-yy86:
+yy91:
++p;
yych = *p;
-yy87:
+yy92:
if (yybm[0+yych] & 128) {
- goto yy86;
+ goto yy91;
}
- goto yy83;
+ goto yy88;
}
}
@@ -615,29 +638,29 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
yych = *p;
if (yych <= ' ') {
if (yych <= '\n') {
- if (yych <= 0x00) goto yy96;
- if (yych >= '\n') goto yy92;
+ if (yych <= 0x00) goto yy101;
+ if (yych >= '\n') goto yy97;
} else {
- if (yych == '\r') goto yy98;
- if (yych >= ' ') goto yy92;
+ if (yych == '\r') goto yy103;
+ if (yych >= ' ') goto yy97;
}
} else {
if (yych <= '9') {
- if (yych == '$') goto yy94;
+ if (yych == '$') goto yy99;
} else {
- if (yych <= ':') goto yy92;
- if (yych == '|') goto yy92;
+ if (yych <= ':') goto yy97;
+ if (yych == '|') goto yy97;
}
}
++p;
yych = *p;
- goto yy121;
-yy91:
+ goto yy126;
+yy96:
{
eval->AddText(StringPiece(start, p - start));
continue;
}
-yy92:
+yy97:
++p;
{
if (path) {
@@ -650,137 +673,137 @@ yy92:
continue;
}
}
-yy94:
+yy99:
++p;
if ((yych = *p) <= '/') {
if (yych <= ' ') {
- if (yych == '\n') goto yy110;
- if (yych <= 0x1F) goto yy99;
- goto yy101;
+ if (yych == '\n') goto yy115;
+ if (yych <= 0x1F) goto yy104;
+ goto yy106;
} else {
if (yych <= '$') {
- if (yych <= '#') goto yy99;
- goto yy103;
+ if (yych <= '#') goto yy104;
+ goto yy108;
} else {
- if (yych == '-') goto yy105;
- goto yy99;
+ if (yych == '-') goto yy110;
+ goto yy104;
}
}
} else {
if (yych <= '^') {
if (yych <= ':') {
- if (yych <= '9') goto yy105;
- goto yy107;
+ if (yych <= '9') goto yy110;
+ goto yy112;
} else {
- if (yych <= '@') goto yy99;
- if (yych <= 'Z') goto yy105;
- goto yy99;
+ if (yych <= '@') goto yy104;
+ if (yych <= 'Z') goto yy110;
+ goto yy104;
}
} else {
if (yych <= '`') {
- if (yych <= '_') goto yy105;
- goto yy99;
+ if (yych <= '_') goto yy110;
+ goto yy104;
} else {
- if (yych <= 'z') goto yy105;
- if (yych <= '{') goto yy109;
- goto yy99;
+ if (yych <= 'z') goto yy110;
+ if (yych <= '{') goto yy114;
+ goto yy104;
}
}
}
-yy95:
+yy100:
{
last_token_ = start;
return Error(DescribeLastError(), err);
}
-yy96:
+yy101:
++p;
{
last_token_ = start;
return Error("unexpected EOF", err);
}
-yy98:
+yy103:
yych = *++p;
- goto yy95;
-yy99:
+ goto yy100;
+yy104:
++p;
-yy100:
+yy105:
{
last_token_ = start;
return Error("bad $-escape (literal $ must be written as $$)", err);
}
-yy101:
+yy106:
++p;
{
eval->AddText(StringPiece(" ", 1));
continue;
}
-yy103:
+yy108:
++p;
{
eval->AddText(StringPiece("$", 1));
continue;
}
-yy105:
+yy110:
++p;
yych = *p;
- goto yy119;
-yy106:
+ goto yy124;
+yy111:
{
eval->AddSpecial(StringPiece(start + 1, p - start - 1));
continue;
}
-yy107:
+yy112:
++p;
{
eval->AddText(StringPiece(":", 1));
continue;
}
-yy109:
+yy114:
yych = *(q = ++p);
if (yybm[0+yych] & 32) {
- goto yy113;
+ goto yy118;
}
- goto yy100;
-yy110:
+ goto yy105;
+yy115:
++p;
yych = *p;
if (yybm[0+yych] & 16) {
- goto yy110;
+ goto yy115;
}
{
continue;
}
-yy113:
+yy118:
++p;
yych = *p;
if (yybm[0+yych] & 32) {
- goto yy113;
+ goto yy118;
}
- if (yych == '}') goto yy116;
+ if (yych == '}') goto yy121;
p = q;
- goto yy100;
-yy116:
+ goto yy105;
+yy121:
++p;
{
eval->AddSpecial(StringPiece(start + 2, p - start - 3));
continue;
}
-yy118:
+yy123:
++p;
yych = *p;
-yy119:
+yy124:
if (yybm[0+yych] & 64) {
- goto yy118;
+ goto yy123;
}
- goto yy106;
-yy120:
+ goto yy111;
+yy125:
++p;
yych = *p;
-yy121:
+yy126:
if (yybm[0+yych] & 128) {
- goto yy120;
+ goto yy125;
}
- goto yy91;
+ goto yy96;
}
}
diff --git a/src/lexer.h b/src/lexer.h
index 03c59f2..f366556 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -41,6 +41,7 @@ struct Lexer {
NEWLINE,
PIPE,
PIPE2,
+ POOL,
RULE,
SUBNINJA,
TEOF,
diff --git a/src/lexer.in.cc b/src/lexer.in.cc
index 7ae9c61..93d5540 100644
--- a/src/lexer.in.cc
+++ b/src/lexer.in.cc
@@ -82,6 +82,7 @@ const char* Lexer::TokenName(Token t) {
case NEWLINE: return "newline";
case PIPE2: return "'||'";
case PIPE: return "'|'";
+ case POOL: return "'pool'";
case RULE: return "'rule'";
case SUBNINJA: return "'subninja'";
case TEOF: return "eof";
@@ -135,6 +136,7 @@ Lexer::Token Lexer::ReadToken() {
[ ]*[\n] { token = NEWLINE; break; }
[ ]+ { token = INDENT; break; }
"build" { token = BUILD; break; }
+ "pool" { token = POOL; break; }
"rule" { token = RULE; break; }
"default" { token = DEFAULT; break; }
"=" { token = EQUALS; break; }
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 405e244..2d052b5 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -47,6 +47,10 @@ bool ManifestParser::Parse(const string& filename, const string& input,
for (;;) {
Lexer::Token token = lexer_.ReadToken();
switch (token) {
+ case Lexer::POOL:
+ if (!ParsePool(err))
+ return false;
+ break;
case Lexer::BUILD:
if (!ParseEdge(err))
return false;
@@ -91,6 +95,44 @@ bool ManifestParser::Parse(const string& filename, const string& input,
return false; // not reached
}
+
+bool ManifestParser::ParsePool(string* err) {
+ string name;
+ if (!lexer_.ReadIdent(&name))
+ return lexer_.Error("expected pool name", err);
+
+ if (!ExpectToken(Lexer::NEWLINE, err))
+ return false;
+
+ if (state_->LookupPool(name) != NULL)
+ return lexer_.Error("duplicate pool '" + name + "'", err);
+
+ int depth = -1;
+
+ while (lexer_.PeekToken(Lexer::INDENT)) {
+ string key;
+ EvalString value;
+ if (!ParseLet(&key, &value, err))
+ return false;
+
+ if (key == "depth") {
+ string depth_string = value.Evaluate(env_);
+ depth = atol(depth_string.c_str());
+ if (depth < 0)
+ return lexer_.Error("invalid pool depth", err);
+ } else {
+ return lexer_.Error("unexpected variable '" + key + "'", err);
+ }
+ }
+
+ if (depth < 0)
+ return lexer_.Error("expected 'depth =' line", err);
+
+ state_->AddPool(new Pool(name, depth));
+ return true;
+}
+
+
bool ManifestParser::ParseRule(string* err) {
string name;
if (!lexer_.ReadIdent(&name))
@@ -126,6 +168,8 @@ bool ManifestParser::ParseRule(string* err) {
rule->rspfile_ = value;
} else if (key == "rspfile_content") {
rule->rspfile_content_ = value;
+ } else if (key == "pool") {
+ rule->pool_ = value;
} else {
// Die on other keyvals for now; revisit if we want to add a
// scope here.
@@ -133,8 +177,10 @@ bool ManifestParser::ParseRule(string* err) {
}
}
- if (rule->rspfile_.empty() != rule->rspfile_content_.empty())
- return lexer_.Error("rspfile and rspfile_content need to be both specified", err);
+ if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) {
+ return lexer_.Error("rspfile and rspfile_content need to be both specified",
+ err);
+ }
if (rule->command_.empty())
return lexer_.Error("expected 'command =' line", err);
@@ -252,6 +298,7 @@ bool ManifestParser::ParseEdge(string* err) {
// Default to using outer env.
BindingEnv* env = env_;
+ Pool* pool = NULL;
// But create and fill a nested env if there are variables in scope.
if (lexer_.PeekToken(Lexer::INDENT)) {
@@ -262,11 +309,28 @@ bool ManifestParser::ParseEdge(string* err) {
EvalString val;
if (!ParseLet(&key, &val, err))
return false;
- env->AddBinding(key, val.Evaluate(env_));
+ if (key == "pool") {
+ string pool_name = val.Evaluate(env_);
+ pool = state_->LookupPool(pool_name);
+ if (pool == NULL)
+ return lexer_.Error("undefined pool '" + pool_name + "'", err);
+ } else {
+ env->AddBinding(key, val.Evaluate(env_));
+ }
} while (lexer_.PeekToken(Lexer::INDENT));
}
- Edge* edge = state_->AddEdge(rule);
+ if (pool == NULL) {
+ if (!rule->pool_.empty()) {
+ pool = state_->LookupPool(rule->pool_.Evaluate(env_));
+ if (pool == NULL)
+ return lexer_.Error("cannot resolve pool for this edge.", err);
+ } else {
+ pool = &State::kDefaultPool;
+ }
+ }
+
+ Edge* edge = state_->AddEdge(rule, pool);
edge->env_ = env;
for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
string path = i->Evaluate(env);
diff --git a/src/manifest_parser.h b/src/manifest_parser.h
index a2c6c93..a08e5af 100644
--- a/src/manifest_parser.h
+++ b/src/manifest_parser.h
@@ -50,6 +50,7 @@ private:
bool Parse(const string& filename, const string& input, string* err);
/// Parse various statement types.
+ bool ParsePool(string* err);
bool ParseRule(string* err);
bool ParseLet(string* key, EvalString* val, string* err);
bool ParseEdge(string* err);
diff --git a/src/metrics.h b/src/metrics.h
index f5ac0de..b6da859 100644
--- a/src/metrics.h
+++ b/src/metrics.h
@@ -59,29 +59,27 @@ private:
};
/// Get the current time as relative to some epoch.
-/// Epoch varies between platforms; only useful for measuring elapsed
-/// time.
+/// Epoch varies between platforms; only useful for measuring elapsed time.
int64_t GetTimeMillis();
-
-/// A simple stopwatch which retruns the time
-// in seconds since Restart() was called
-class Stopwatch
-{
-public:
+/// A simple stopwatch which returns the time
+/// in seconds since Restart() was called.
+struct Stopwatch {
+ public:
Stopwatch() : started_(0) {}
- /// Seconds since Restart() call
- double Elapsed() const { return 1e-6 * static_cast<double>(Now() - started_); }
+ /// Seconds since Restart() call.
+ double Elapsed() const {
+ return 1e-6 * static_cast<double>(Now() - started_);
+ }
void Restart() { started_ = Now(); }
-private:
+ private:
uint64_t started_;
uint64_t Now() const;
};
-
/// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top
/// of a function to get timing stats recorded for each call of the function.
#define METRIC_RECORD(name) \
diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc
index 8e440fe..fd9b671 100644
--- a/src/msvc_helper-win32.cc
+++ b/src/msvc_helper-win32.cc
@@ -14,6 +14,7 @@
#include "msvc_helper.h"
+#include <stdio.h>
#include <string.h>
#include <windows.h>
@@ -28,6 +29,21 @@ bool EndsWith(const string& input, const string& needle) {
input.substr(input.size() - needle.size()) == needle);
}
+string Replace(const string& input, const string& find, const string& replace) {
+ string result = input;
+ size_t start_pos = 0;
+ while ((start_pos = result.find(find, start_pos)) != string::npos) {
+ result.replace(start_pos, find.length(), replace);
+ start_pos += replace.length();
+ }
+ return result;
+}
+
+string EscapeForDepfile(const string& path) {
+ // Depfiles don't escape single \.
+ return Replace(path, " ", "\\ ");
+}
+
} // anonymous namespace
// static
@@ -125,7 +141,7 @@ int CLWrapper::Run(const string& command, string* extra_output) {
if (!include.empty()) {
include = IncludesNormalize::Normalize(include, NULL);
if (!IsSystemInclude(include))
- includes_.push_back(include);
+ includes_.insert(include);
} else if (FilterInputFilename(line)) {
// Drop it.
// TODO: if we support compiling multiple output files in a single
@@ -162,3 +178,11 @@ int CLWrapper::Run(const string& command, string* extra_output) {
return exit_code;
}
+
+vector<string> CLWrapper::GetEscapedResult() {
+ vector<string> result;
+ for (set<string>::iterator i = includes_.begin(); i != includes_.end(); ++i) {
+ result.push_back(EscapeForDepfile(*i));
+ }
+ return result;
+}
diff --git a/src/msvc_helper.h b/src/msvc_helper.h
index f623520..102201b 100644
--- a/src/msvc_helper.h
+++ b/src/msvc_helper.h
@@ -13,6 +13,7 @@
// limitations under the License.
#include <string>
+#include <set>
#include <vector>
using namespace std;
@@ -49,6 +50,10 @@ struct CLWrapper {
/// Exposed for testing.
static bool FilterInputFilename(const string& line);
+ /// Fill a vector with the unique'd headers, escaped for output as a .d
+ /// file.
+ vector<string> GetEscapedResult();
+
void* env_block_;
- vector<string> includes_;
+ set<string> includes_;
};
diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc
index 0c8db37..152450e 100644
--- a/src/msvc_helper_main-win32.cc
+++ b/src/msvc_helper_main-win32.cc
@@ -14,6 +14,7 @@
#include "msvc_helper.h"
+#include <stdio.h>
#include <windows.h>
#include "util.h"
@@ -27,7 +28,6 @@ void Usage() {
"usage: ninja -t msvc [options] -- cl.exe /showIncludes /otherArgs\n"
"options:\n"
" -e ENVFILE load environment block from ENVFILE as environment\n"
-" -r BASE normalize paths and make relative to BASE before output\n"
" -o FILE write output dependency information to FILE.d\n"
);
}
@@ -48,7 +48,6 @@ void PushPathIntoEnvironment(const string& env_block) {
int MSVCHelperMain(int argc, char** argv) {
const char* output_filename = NULL;
- const char* relative_to = NULL;
const char* envfile = NULL;
const option kLongOptions[] = {
@@ -56,7 +55,7 @@ int MSVCHelperMain(int argc, char** argv) {
{ NULL, 0, NULL, 0 }
};
int opt;
- while ((opt = getopt_long(argc, argv, "e:o:r:h", kLongOptions, NULL)) != -1) {
+ while ((opt = getopt_long(argc, argv, "e:o:h", kLongOptions, NULL)) != -1) {
switch (opt) {
case 'e':
envfile = optarg;
@@ -64,9 +63,6 @@ int MSVCHelperMain(int argc, char** argv) {
case 'o':
output_filename = optarg;
break;
- case 'r':
- relative_to = optarg;
- break;
case 'h':
default:
Usage();
@@ -105,8 +101,8 @@ int MSVCHelperMain(int argc, char** argv) {
Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str());
}
fprintf(output, "%s: ", output_filename);
- for (vector<string>::iterator i = cl.includes_.begin();
- i != cl.includes_.end(); ++i) {
+ vector<string> headers = cl.GetEscapedResult();
+ for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) {
fprintf(output, "%s\n", i->c_str());
}
fclose(output);
diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc
index 29fefd4..7730425 100644
--- a/src/msvc_helper_test.cc
+++ b/src/msvc_helper_test.cc
@@ -48,7 +48,7 @@ TEST(MSVCHelperTest, Run) {
&output);
ASSERT_EQ("foo\nbar\n", output);
ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("foo.h", cl.includes_[0]);
+ ASSERT_EQ("foo.h", *cl.includes_.begin());
}
TEST(MSVCHelperTest, RunFilenameFilter) {
@@ -70,7 +70,7 @@ TEST(MSVCHelperTest, RunSystemInclude) {
// system headers.
ASSERT_EQ("", output);
ASSERT_EQ(1u, cl.includes_.size());
- ASSERT_EQ("path.h", cl.includes_[0]);
+ ASSERT_EQ("path.h", *cl.includes_.begin());
}
TEST(MSVCHelperTest, EnvBlock) {
@@ -81,3 +81,38 @@ TEST(MSVCHelperTest, EnvBlock) {
cl.Run("cmd /c \"echo foo is %foo%", &output);
ASSERT_EQ("foo is bar\n", output);
}
+
+TEST(MSVCHelperTest, DuplicatedHeader) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: foo.h&&"
+ "echo Note: including file: bar.h&&"
+ "echo Note: including file: foo.h\"",
+ &output);
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, cl.includes_.size());
+}
+
+TEST(MSVCHelperTest, DuplicatedHeaderPathConverted) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: sub/foo.h&&"
+ "echo Note: including file: bar.h&&"
+ "echo Note: including file: sub\\foo.h\"",
+ &output);
+ // We should have dropped one copy of foo.h.
+ ASSERT_EQ("", output);
+ ASSERT_EQ(2u, cl.includes_.size());
+}
+
+TEST(MSVCHelperTest, SpacesInFilename) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo Note: including file: sub\\some sdk\\foo.h",
+ &output);
+ ASSERT_EQ("", output);
+ vector<string> headers = cl.GetEscapedResult();
+ ASSERT_EQ(1u, headers.size());
+ ASSERT_EQ("sub\\some\\ sdk\\foo.h", headers[0]);
+}
diff --git a/src/ninja.cc b/src/ninja.cc
index 5a3c530..08d4b14 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -49,7 +49,7 @@ namespace {
/// The version number of the current Ninja release. This will always
/// be "git" on trunk.
-const char* kVersion = "1.0.0";
+const char* kVersion = "1.1.0";
/// Global information passed into subtools.
struct Globals {
@@ -140,7 +140,7 @@ int GuessParallelism() {
/// An implementation of ManifestParser::FileReader that actually reads
/// the file.
struct RealFileReader : public ManifestParser::FileReader {
- bool ReadFile(const string& path, string* content, string* err) {
+ virtual bool ReadFile(const string& path, string* content, string* err) {
return ::ReadFile(path, content, err) == 0;
}
};
@@ -168,6 +168,50 @@ bool RebuildManifest(Builder* builder, const char* input_file, string* err) {
return node->dirty();
}
+Node* CollectTarget(State* state, const char* cpath, string* err) {
+ string path = cpath;
+ if (!CanonicalizePath(&path, err))
+ return NULL;
+
+ // Special syntax: "foo.cc^" means "the first output of foo.cc".
+ bool first_dependent = false;
+ if (!path.empty() && path[path.size() - 1] == '^') {
+ path.resize(path.size() - 1);
+ first_dependent = true;
+ }
+
+ Node* node = state->LookupNode(path);
+ if (node) {
+ if (first_dependent) {
+ if (node->out_edges().empty()) {
+ *err = "'" + path + "' has no out edge";
+ return NULL;
+ }
+ Edge* edge = node->out_edges()[0];
+ if (edge->outputs_.empty()) {
+ edge->Dump();
+ Fatal("edge has no outputs");
+ }
+ node = edge->outputs_[0];
+ }
+ return node;
+ } else {
+ *err = "unknown target '" + path + "'";
+
+ if (path == "clean") {
+ *err += ", did you mean 'ninja -t clean'?";
+ } else if (path == "help") {
+ *err += ", did you mean 'ninja -h'?";
+ } else {
+ Node* suggestion = state->SpellcheckNode(path);
+ if (suggestion) {
+ *err += ", did you mean '" + suggestion->path() + "'?";
+ }
+ }
+ return NULL;
+ }
+}
+
bool CollectTargetsFromArgs(State* state, int argc, char* argv[],
vector<Node*>* targets, string* err) {
if (argc == 0) {
@@ -176,47 +220,10 @@ bool CollectTargetsFromArgs(State* state, int argc, char* argv[],
}
for (int i = 0; i < argc; ++i) {
- string path = argv[i];
- if (!CanonicalizePath(&path, err))
+ Node* node = CollectTarget(state, argv[i], err);
+ if (node == NULL)
return false;
-
- // Special syntax: "foo.cc^" means "the first output of foo.cc".
- bool first_dependent = false;
- if (!path.empty() && path[path.size() - 1] == '^') {
- path.resize(path.size() - 1);
- first_dependent = true;
- }
-
- Node* node = state->LookupNode(path);
- if (node) {
- if (first_dependent) {
- if (node->out_edges().empty()) {
- *err = "'" + path + "' has no out edge";
- return false;
- }
- Edge* edge = node->out_edges()[0];
- if (edge->outputs_.empty()) {
- edge->Dump();
- Fatal("edge has no outputs");
- }
- node = edge->outputs_[0];
- }
- targets->push_back(node);
- } else {
- *err = "unknown target '" + path + "'";
-
- if (path == "clean") {
- *err += ", did you mean 'ninja -t clean'?";
- } else if (path == "help") {
- *err += ", did you mean 'ninja -h'?";
- } else {
- Node* suggestion = state->SpellcheckNode(path);
- if (suggestion) {
- *err += ", did you mean '" + suggestion->path() + "'?";
- }
- }
- return false;
- }
+ targets->push_back(node);
}
return true;
}
@@ -244,19 +251,14 @@ int ToolQuery(Globals* globals, int argc, char* argv[]) {
return 1;
}
for (int i = 0; i < argc; ++i) {
- Node* node = globals->state->LookupNode(argv[i]);
+ string err;
+ Node* node = CollectTarget(globals->state, argv[i], &err);
if (!node) {
- Node* suggestion = globals->state->SpellcheckNode(argv[i]);
- if (suggestion) {
- printf("%s unknown, did you mean %s?\n",
- argv[i], suggestion->path().c_str());
- } else {
- printf("%s unknown\n", argv[i]);
- }
+ Error("%s", err.c_str());
return 1;
}
- printf("%s:\n", argv[i]);
+ printf("%s:\n", node->path().c_str());
if (Edge* edge = node->in_edge()) {
printf(" input: %s\n", edge->rule_->name().c_str());
for (int in = 0; in < (int)edge->inputs_.size(); in++) {
@@ -292,7 +294,7 @@ int ToolBrowse(Globals* globals, int argc, char* argv[]) {
}
#endif // _WIN32
-#if defined(WIN32)
+#if defined(_WIN32)
int ToolMSVC(Globals* globals, int argc, char* argv[]) {
// Reset getopt: push one argument onto the front of argv, reset optind.
argc++;
@@ -537,7 +539,7 @@ int ChooseTool(const string& tool_name, const Tool** tool_out) {
{ "browse", "browse dependency graph in a web browser",
Tool::RUN_AFTER_LOAD, ToolBrowse },
#endif
-#if defined(WIN32)
+#if defined(_WIN32)
{ "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)",
Tool::RUN_AFTER_FLAGS, ToolMSVC },
#endif
@@ -682,6 +684,9 @@ int RunBuild(Builder* builder, int argc, char** argv) {
if (!builder->Build(&err)) {
printf("ninja: build stopped: %s.\n", err.c_str());
+ if (err.find("interrupted by user") != string::npos) {
+ return 2;
+ }
return 1;
}
@@ -740,7 +745,7 @@ int NinjaMain(int argc, char** argv) {
int opt;
while (tool_name.empty() &&
- (opt = getopt_long(argc, argv, "d:f:hj:k:l:nt:vC:V", kLongOptions,
+ (opt = getopt_long(argc, argv, "d:f:j:k:l:nt:vC:h", kLongOptions,
NULL)) != -1) {
switch (opt) {
case 'd':
@@ -753,14 +758,6 @@ int NinjaMain(int argc, char** argv) {
case 'j':
config.parallelism = atoi(optarg);
break;
- case 'l': {
- char* end;
- double value = strtod(optarg, &end);
- if (end == optarg)
- Fatal("-l parameter not numeric: did you mean -l 0.0?");
- config.max_load_average = value;
- break;
- }
case 'k': {
char* end;
int value = strtol(optarg, &end, 10);
@@ -773,15 +770,23 @@ int NinjaMain(int argc, char** argv) {
config.failures_allowed = value > 0 ? value : INT_MAX;
break;
}
+ case 'l': {
+ char* end;
+ double value = strtod(optarg, &end);
+ if (end == optarg)
+ Fatal("-l parameter not numeric: did you mean -l 0.0?");
+ config.max_load_average = value;
+ break;
+ }
case 'n':
config.dry_run = true;
break;
- case 'v':
- config.verbosity = BuildConfig::VERBOSE;
- break;
case 't':
tool_name = optarg;
break;
+ case 'v':
+ config.verbosity = BuildConfig::VERBOSE;
+ break;
case 'C':
working_dir = optarg;
break;
@@ -825,7 +830,6 @@ int NinjaMain(int argc, char** argv) {
bool rebuilt_manifest = false;
reload:
- RealDiskInterface disk_interface;
RealFileReader file_reader;
ManifestParser parser(globals.state, &file_reader);
string err;
@@ -838,6 +842,7 @@ reload:
return tool->func(&globals, argc, argv);
BuildLog build_log;
+ RealDiskInterface disk_interface;
if (!OpenLog(&build_log, &globals, &disk_interface))
return 1;
diff --git a/src/state.cc b/src/state.cc
index 4c7168b..bb0cc15 100644
--- a/src/state.cc
+++ b/src/state.cc
@@ -22,10 +22,49 @@
#include "metrics.h"
#include "util.h"
+
+void Pool::EdgeScheduled(const Edge& edge) {
+ if (depth_ != 0)
+ current_use_ += edge.weight();
+}
+
+void Pool::EdgeFinished(const Edge& edge) {
+ if (depth_ != 0)
+ current_use_ -= edge.weight();
+}
+
+void Pool::DelayEdge(Edge* edge) {
+ assert(depth_ != 0);
+ delayed_.push_back(edge);
+}
+
+void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) {
+ while (!delayed_.empty()) {
+ Edge* edge = delayed_.front();
+ if (current_use_ + edge->weight() > depth_)
+ break;
+ delayed_.pop_front();
+ ready_queue->insert(edge);
+ EdgeScheduled(*edge);
+ }
+}
+
+void Pool::Dump() const {
+ printf("%s (%d/%d) ->\n", name_.c_str(), current_use_, depth_);
+ for (deque<Edge*>::const_iterator it = delayed_.begin();
+ it != delayed_.end(); ++it)
+ {
+ printf("\t");
+ (*it)->Dump();
+ }
+}
+
+Pool State::kDefaultPool("", 0);
const Rule State::kPhonyRule("phony");
State::State() {
AddRule(&kPhonyRule);
+ AddPool(&kDefaultPool);
}
void State::AddRule(const Rule* rule) {
@@ -40,9 +79,22 @@ const Rule* State::LookupRule(const string& rule_name) {
return i->second;
}
-Edge* State::AddEdge(const Rule* rule) {
+void State::AddPool(Pool* pool) {
+ assert(LookupPool(pool->name()) == NULL);
+ pools_[pool->name()] = pool;
+}
+
+Pool* State::LookupPool(const string& pool_name) {
+ map<string, Pool*>::iterator i = pools_.find(pool_name);
+ if (i == pools_.end())
+ return NULL;
+ return i->second;
+}
+
+Edge* State::AddEdge(const Rule* rule, Pool* pool) {
Edge* edge = new Edge();
edge->rule_ = rule;
+ edge->pool_ = pool;
edge->env_ = &bindings_;
edges_.push_back(edge);
return edge;
@@ -146,4 +198,14 @@ void State::Dump() {
node->status_known() ? (node->dirty() ? "dirty" : "clean")
: "unknown");
}
+ if (!pools_.empty()) {
+ printf("resource_pools:\n");
+ for (map<string, Pool*>::const_iterator it = pools_.begin();
+ it != pools_.end(); ++it)
+ {
+ if (!it->second->name().empty()) {
+ it->second->Dump();
+ }
+ }
+ }
}
diff --git a/src/state.h b/src/state.h
index 026acf3..918fe09 100644
--- a/src/state.h
+++ b/src/state.h
@@ -16,6 +16,8 @@
#define NINJA_STATE_H_
#include <map>
+#include <deque>
+#include <set>
#include <string>
#include <vector>
using namespace std;
@@ -27,8 +29,59 @@ struct Edge;
struct Node;
struct Rule;
+/// A pool for delayed edges.
+/// Pools are scoped to a State. Edges within a State will share Pools. A Pool
+/// will keep a count of the total 'weight' of the currently scheduled edges. If
+/// a Plan attempts to schedule an Edge which would cause the total weight to
+/// exceed the depth of the Pool, the Pool will enque the Edge instead of
+/// allowing the Plan to schedule it. The Pool will relinquish queued Edges when
+/// the total scheduled weight diminishes enough (i.e. when a scheduled edge
+/// completes).
+struct Pool {
+ explicit Pool(const string& name, int depth)
+ : name_(name), current_use_(0), depth_(depth) { }
+
+ // A depth of 0 is infinite
+ bool is_valid() const { return depth_ >= 0; }
+ int depth() const { return depth_; }
+ const string& name() const { return name_; }
+
+ /// true if the Pool might delay this edge
+ bool ShouldDelayEdge() const { return depth_ != 0; }
+
+ /// informs this Pool that the given edge is committed to be run.
+ /// Pool will count this edge as using resources from this pool.
+ void EdgeScheduled(const Edge& edge);
+
+ /// informs this Pool that the given edge is no longer runnable, and should
+ /// relinquish its resources back to the pool
+ void EdgeFinished(const Edge& edge);
+
+ /// adds the given edge to this Pool to be delayed.
+ void DelayEdge(Edge* edge);
+
+ /// Pool will add zero or more edges to the ready_queue
+ void RetrieveReadyEdges(set<Edge*>* ready_queue);
+
+ /// Dump the Pool and its edges (useful for debugging).
+ void Dump() const;
+
+private:
+ int UnitsWaiting() { return delayed_.size(); }
+
+ string name_;
+
+ /// |current_use_| is the total of the weights of the edges which are
+ /// currently scheduled in the Plan (i.e. the edges in Plan::ready_).
+ int current_use_;
+ int depth_;
+
+ deque<Edge*> delayed_;
+};
+
/// Global state (file status, loaded rules) for a single run.
struct State {
+ static Pool kDefaultPool;
static const Rule kPhonyRule;
State();
@@ -36,7 +89,10 @@ struct State {
void AddRule(const Rule* rule);
const Rule* LookupRule(const string& rule_name);
- Edge* AddEdge(const Rule* rule);
+ void AddPool(Pool* pool);
+ Pool* LookupPool(const string& pool_name);
+
+ Edge* AddEdge(const Rule* rule, Pool* pool);
Node* GetNode(StringPiece path);
Node* LookupNode(StringPiece path);
@@ -50,7 +106,7 @@ struct State {
/// state where we haven't yet examined the disk for dirty state.
void Reset();
- /// Dump the nodes (useful for debugging).
+ /// Dump the nodes and Pools (useful for debugging).
void Dump();
/// @return the root node(s) of the graph. (Root nodes have no output edges).
@@ -65,6 +121,9 @@ struct State {
/// All the rules used in the graph.
map<string, const Rule*> rules_;
+ /// All the pools used in the graph.
+ map<string, Pool*> pools_;
+
/// All the edges of the graph.
vector<Edge*> edges_;
diff --git a/src/state_test.cc b/src/state_test.cc
index bc24edd..26177ff 100644
--- a/src/state_test.cc
+++ b/src/state_test.cc
@@ -32,7 +32,7 @@ TEST(State, Basic) {
rule->set_command(command);
state.AddRule(rule);
- Edge* edge = state.AddEdge(rule);
+ Edge* edge = state.AddEdge(rule, &State::kDefaultPool);
state.AddIn(edge, "in1");
state.AddIn(edge, "in2");
state.AddOut(edge, "out");
diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc
index 1c47fd1..8f1a04e 100644
--- a/src/subprocess-posix.cc
+++ b/src/subprocess-posix.cc
@@ -76,7 +76,7 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) {
break;
// Open /dev/null over stdin.
- int devnull = open("/dev/null", O_WRONLY);
+ int devnull = open("/dev/null", O_RDONLY);
if (devnull < 0)
break;
if (dup2(devnull, 0) < 0)
diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc
index 4b103a5..1b230b6 100644
--- a/src/subprocess-win32.cc
+++ b/src/subprocess-win32.cc
@@ -101,14 +101,17 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) {
NULL, NULL,
&startup_info, &process_info)) {
DWORD error = GetLastError();
- if (error == ERROR_FILE_NOT_FOUND) { // file (program) not found error is treated as a normal build action failure
+ if (error == ERROR_FILE_NOT_FOUND) {
+ // File (program) not found error is treated as a normal build
+ // action failure.
if (child_pipe)
CloseHandle(child_pipe);
CloseHandle(pipe_);
CloseHandle(nul);
pipe_ = NULL;
// child_ is already NULL;
- buf_ = "CreateProcess failed: The system cannot find the file specified.\n";
+ buf_ = "CreateProcess failed: The system cannot find the file "
+ "specified.\n";
return true;
} else {
Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal
diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc
index d89525e..c3175da 100644
--- a/src/subprocess_test.cc
+++ b/src/subprocess_test.cc
@@ -64,7 +64,8 @@ TEST_F(SubprocessTest, NoSuchCommand) {
EXPECT_EQ(ExitFailure, subproc->Finish());
EXPECT_NE("", subproc->GetOutput());
#ifdef _WIN32
- ASSERT_EQ("CreateProcess failed: The system cannot find the file specified.\n", subproc->GetOutput());
+ ASSERT_EQ("CreateProcess failed: The system cannot find the file "
+ "specified.\n", subproc->GetOutput());
#endif
}
@@ -179,3 +180,18 @@ TEST_F(SubprocessTest, SetWithLots) {
ASSERT_EQ(kNumProcs, subprocs_.finished_.size());
}
#endif // linux
+
+// TODO: this test could work on Windows, just not sure how to simply
+// read stdin.
+#ifndef _WIN32
+// Verify that a command that attempts to read stdin correctly thinks
+// that stdin is closed.
+TEST_F(SubprocessTest, ReadStdin) {
+ Subprocess* subproc = subprocs_.Add("cat -");
+ while (!subproc->Done()) {
+ subprocs_.DoWork();
+ }
+ ASSERT_EQ(ExitSuccess, subproc->Finish());
+ ASSERT_EQ(1u, subprocs_.finished_.size());
+}
+#endif // _WIN32
diff --git a/src/util.cc b/src/util.cc
index 0feb99d..4b2900f 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -155,7 +155,7 @@ bool CanonicalizePath(char* path, size_t* len, string* err) {
}
if (component_count == kMaxPathComponents)
- Fatal("path has too many components");
+ Fatal("path has too many components : %s", path);
components[component_count] = dst;
++component_count;
diff --git a/src/util.h b/src/util.h
index 6c142c6..2b59283 100644
--- a/src/util.h
+++ b/src/util.h
@@ -49,7 +49,8 @@ void SetCloseOnExec(int fd);
/// Given a misspelled string and a list of correct spellings, returns
/// the closest match or NULL if there is no close enough match.
-const char* SpellcheckStringV(const string& text, const vector<const char*>& words);
+const char* SpellcheckStringV(const string& text,
+ const vector<const char*>& words);
/// Like SpellcheckStringV, but takes a NULL-terminated list.
const char* SpellcheckString(const string& text, ...);