diff options
-rw-r--r-- | RELEASING | 10 | ||||
-rwxr-xr-x | bootstrap.py | 5 | ||||
-rwxr-xr-x | configure.py | 28 | ||||
-rw-r--r-- | doc/docbook.xsl | 17 | ||||
-rw-r--r-- | doc/manual.asciidoc | 183 | ||||
-rw-r--r-- | doc/style.css | 29 | ||||
-rw-r--r-- | misc/packaging/ninja.spec | 8 | ||||
-rwxr-xr-x | src/browse.py | 3 | ||||
-rw-r--r-- | src/build.cc | 51 | ||||
-rw-r--r-- | src/build.h | 1 | ||||
-rw-r--r-- | src/build_test.cc | 74 | ||||
-rw-r--r-- | src/clean.cc | 6 | ||||
-rw-r--r-- | src/depfile_parser.cc | 97 | ||||
-rw-r--r-- | src/depfile_parser.in.cc | 9 | ||||
-rw-r--r-- | src/depfile_parser_test.cc | 8 | ||||
-rw-r--r-- | src/eval_env.cc | 16 | ||||
-rw-r--r-- | src/eval_env.h | 12 | ||||
-rw-r--r-- | src/graph.cc | 85 | ||||
-rw-r--r-- | src/graph.h | 43 | ||||
-rw-r--r-- | src/graph_test.cc | 35 | ||||
-rw-r--r-- | src/manifest_parser.cc | 88 | ||||
-rw-r--r-- | src/manifest_parser_test.cc | 29 | ||||
-rw-r--r-- | src/msvc_helper_main-win32.cc | 37 | ||||
-rw-r--r-- | src/ninja.cc | 83 | ||||
-rw-r--r-- | src/state.cc | 23 | ||||
-rw-r--r-- | src/state.h | 13 | ||||
-rw-r--r-- | src/state_test.cc | 4 | ||||
-rw-r--r-- | src/util.cc | 6 | ||||
-rw-r--r-- | src/util.h | 2 | ||||
-rw-r--r-- | src/version.cc | 57 | ||||
-rw-r--r-- | src/version.h | 32 |
31 files changed, 760 insertions, 334 deletions
diff --git a/RELEASING b/RELEASING new file mode 100644 index 0000000..0593e6d --- /dev/null +++ b/RELEASING @@ -0,0 +1,10 @@ +Notes to myself on all the steps to make for a Ninja release. + +1. git checkout release; git merge master +2. fix version number in source (it will likely conflict in the above) +3. fix version in doc/manual.asciidoc +4. grep doc/manual.asciidoc for XXX, fix version references +5. rebuild manual, put in place on website +6. commit, tag, push +7. construct release notes from prior notes + credits: git shortlog -s --no-merges REV.. diff --git a/bootstrap.py b/bootstrap.py index a847df9..fcf1a20 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -85,7 +85,9 @@ if options.windows: vcdir = os.environ.get('VCINSTALLDIR') if vcdir: if options.x64: - cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')] + cl = [os.path.join(vcdir, 'bin', 'x86_amd64', 'cl.exe')] + if not os.path.exists(cl[0]): + cl = [os.path.join(vcdir, 'bin', 'amd64', 'cl.exe')] else: cl = [os.path.join(vcdir, 'bin', 'cl.exe')] args = cl + ['/nologo', '/EHsc', '/DNOMINMAX'] @@ -96,6 +98,7 @@ else: '-DNINJA_BOOTSTRAP']) if options.windows: cflags.append('-D_WIN32_WINNT=0x0501') + conf_args.append("--platform=mingw") if options.x64: cflags.append('-m64') args.extend(cflags) diff --git a/configure.py b/configure.py index 9391a68..10c6994 100755 --- a/configure.py +++ b/configure.py @@ -279,7 +279,8 @@ for name in ['build', 'manifest_parser', 'metrics', 'state', - 'util']: + 'util', + 'version']: objs += cxx(name) if platform in ('mingw', 'windows'): for name in ['subprocess-win32', @@ -315,7 +316,7 @@ all_targets += ninja n.comment('Tests all build into ninja_test executable.') variables = [] -test_cflags = cflags[:] +test_cflags = cflags + ['-DGTEST_HAS_RTTI=0'] test_ldflags = None test_libs = libs objs = [] @@ -335,14 +336,12 @@ if options.with_gtest: variables=[('cflags', gtest_cflags)]) test_cflags.append('-I%s' % os.path.join(path, 'include')) -elif platform == 'windows': - test_libs.extend(['gtest_main.lib', 'gtest.lib']) else: - test_cflags.append('-DGTEST_HAS_RTTI=0') - test_libs.extend(['-lgtest_main', '-lgtest']) - -if test_cflags == cflags: - test_cflags = None + # Use gtest from system. + if platform == 'windows': + test_libs.extend(['gtest_main.lib', 'gtest.lib']) + else: + test_libs.extend(['-lgtest_main', '-lgtest']) n.variable('test_cflags', test_cflags) for name in ['build_log_test', @@ -398,9 +397,14 @@ n.newline() n.comment('Generate the manual using asciidoc.') n.rule('asciidoc', - command='asciidoc -a toc -a max-width=45em -o $out $in', - description='ASCIIDOC $in') -manual = n.build(doc('manual.html'), 'asciidoc', doc('manual.asciidoc')) + command='asciidoc -b docbook -d book -o $out $in', + description='ASCIIDOC $out') +n.rule('xsltproc', + command='xsltproc --nonet doc/docbook.xsl $in > $out', + description='XSLTPROC $out') +xml = n.build(built('manual.xml'), 'asciidoc', doc('manual.asciidoc')) +manual = n.build(doc('manual.html'), 'xsltproc', xml, + implicit=doc('style.css')) n.build('manual', 'phony', order_only=manual) n.newline() diff --git a/doc/docbook.xsl b/doc/docbook.xsl new file mode 100644 index 0000000..8afdc8c --- /dev/null +++ b/doc/docbook.xsl @@ -0,0 +1,17 @@ +<!-- This soup of XML is the minimum customization necessary to make the + autogenerated manual look ok. --> +<!DOCTYPE xsl:stylesheet [ +<!ENTITY css SYSTEM "style.css"> +]> +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + version='1.0'> + <xsl:import href="http://docbook.sourceforge.net/release/xsl/current/html/docbook.xsl"/> + <xsl:template name="user.head.content"><style>&css;</style></xsl:template> + <xsl:template name="body.attributes"></xsl:template> + <xsl:param name="generate.toc" select="'book toc'"/> + <xsl:param name="chapter.autolabel" select="0" /> + <xsl:param name="toc.list.type">ul</xsl:param> + + <xsl:output method="html" encoding="utf-8" indent="no" + doctype-public=""/> +</xsl:stylesheet> diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 674231c..688ec93 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -210,20 +210,23 @@ you don't need to pass `-j`.) Environment variables ~~~~~~~~~~~~~~~~~~~~~ -Ninja supports one environment variable to control its behavior. +Ninja supports one environment variable to control its behavior: +`NINJA_STATUS`, the progress status printed before the rule being run. -`NINJA_STATUS`:: The progress status printed before the rule being run. Several placeholders are available: -* `%s`: The number of started edges. -* `%t`: The total number of edges that must be run to complete the build. -* `%p`: The percentage of started edges. -* `%r`: The number of currently running edges. -* `%u`: The number of remaining edges to start. -* `%f`: The number of finished edges. -* `%o`: Overall rate of finished edges per second -* `%c`: Current rate of finished edges per second (average over builds specified by -j or its default) -* `%%`: A plain `%` character. -* The default progress status is `"[%s/%t] "` (note the trailing space + +`%s`:: The number of started edges. +`%t`:: The total number of edges that must be run to complete the build. +`%p`:: The percentage of started edges. +`%r`:: The number of currently running edges. +`%u`:: The number of remaining edges to start. +`%f`:: The number of finished edges. +`%o`:: Overall rate of finished edges per second +`%c`:: Current rate of finished edges per second (average over builds specified by -j or its default) +`%e`:: Elapsed time in seconds. _(Available since Ninja 1.2.)_ +`%%`:: A plain `%` character. + +The default progress status is `"[%s/%t] "` (note the trailing space to separate from the build rule). Another example of possible progress status could be `"[%u/%r/%f] "`. @@ -264,10 +267,6 @@ several times. If used like this +ninja -t targets all+ it prints all the targets available without indentation and it is faster than the _depth_ mode. -`rules`:: output the list of all rules with their description if they have -one. It can be used to know which rule name to pass to -+ninja -t targets rule _name_+. - `commands`:: given a list of targets, print a list of commands which, if executed in order, may be used to rebuild those targets, assuming that all output files are out of date. @@ -286,6 +285,12 @@ Files created but not referenced in the graph are not removed. This tool takes in account the +-v+ and the +-n+ options (note that +-n+ implies +-v+). +`compdb`:: given a list of rules, each of which is expected to be a +C family language compiler rule whose first input is the name of the +source file, prints on standard output a compilation database in the +http://clang.llvm.org/docs/JSONCompilationDatabase.html[JSON format] expected +by the Clang tooling interface. +_Available since Ninja 1.2._ Writing your own Ninja files @@ -359,17 +364,11 @@ consisting of the `rule` keyword and a name for the rule. Then follows an indented set of `variable = value` lines. The basic example above declares a new rule named `cc`, along with the -command to run. (In the context of a rule, the `command` variable is -special and defines the command to run. A full list of special -variables is provided in <<ref_rule,the reference>>.) - -Within the context of a rule, three additional special variables are -available: `$in` expands to the list of input files (`foo.c`) and -`$out` to the output file (`foo.o`) for the command. For use with -`$rspfile_content`, there is also `$in_newline`, which is the same as -`$in`, except that multiple inputs are separated by `\n`, rather than -spaces. - +command to run. In the context of a rule, the `command` variable +defines the command to run, `$in` expands to the list of +input files (`foo.c`), and `$out` to the output files (`foo.o`) for the +command. A full list of special variables is provided in +<<ref_rule,the reference>>. Build statements ~~~~~~~~~~~~~~~~ @@ -400,7 +399,7 @@ rule cc # If left unspecified, builds get the outer $cflags. build foo.o: cc foo.c -# But you can can shadow variables like cflags for a particular build. +# But you can shadow variables like cflags for a particular build. build special.o: cc special.c cflags = -Wall @@ -425,9 +424,7 @@ before building the targets requested by the user. Pools ~~~~~ -*Note: pools were added as an experiment and may be removed in a future -version of Ninja. Are they useful for you? Let us know on the mailing -list.* +_Available since Ninja 1.1._ Pools allow you to allocate one or more rules or edges a finite number of concurrent jobs which is more tightly restricted than the default @@ -541,6 +538,7 @@ This causes Ninja to build the `foo`, `bar` and `baz` targets by default. +[[ref_log]] The Ninja log ~~~~~~~~~~~~~ @@ -554,6 +552,36 @@ If you provide a variable named `builddir` in the outermost scope, `.ninja_log` will be kept in that directory instead. +[[ref_versioning]] +Version compatibility +~~~~~~~~~~~~~~~~~~~~~ + +_Available since Ninja 1.XXX._ + +Ninja version labels follow the standard major.minor.patch format, +where the major version is increased on backwards-incompatible +syntax/behavioral changes and the minor version is increased on new +behaviors. Your `build.ninja` may declare a variable named +`ninja_required_version` that asserts the minimum Ninja version +required to use the generated file. For example, + +----- +ninja_required_version = 1.1 +----- + +declares that the build file relies on some feature that was +introduced in Ninja 1.1 (perhaps the `pool` syntax), and that +Ninja 1.1 or greater must be used to build. Unlike other Ninja +variables, this version requirement is checked immediately when +the variable is encountered in parsing, so it's best to put it +at the top of the build file. + +Ninja always warns if the major versions of Ninja and the +`ninja_required_version` don't match; a major version change hasn't +come up yet so it's difficult to predict what behavior might be +required. + + Ninja file reference -------------------- @@ -608,7 +636,7 @@ across a line break). paths, where a space would otherwise separate filenames. See below.) `$:` :: a colon. (This is only necessary in `build` lines, where a colon -would otherwise terminate the list of inputs.) +would otherwise terminate the list of outputs.) `$$`:: a literal `$`. @@ -639,9 +667,22 @@ line. If a line is indented more than the previous one, it's considered part of its parent's scope; if it is indented less than the previous one, it closes the previous scope. +Top-level variables +~~~~~~~~~~~~~~~~~~~ + +Two variables are significant when declared in the outermost file scope. + +`builddir`:: a directory for some Ninja output files. See <<ref_log,the + discussion of the build log>>. (You can also store other build output + in this directory.) + +`ninja_required_version`:: the minimum version of Ninja required to process + the build correctly. See <<ref_versioning,the discussion of versioning>>. + + +[[ref_rule]] Rule variables ~~~~~~~~~~~~~~ -[[ref_rule]] A `rule` block contains a list of `key = value` declarations that affect the processing of the rule. Here is a full list of special @@ -651,7 +692,7 @@ keys. $variables are expanded) is passed directly to `sh -c` without interpretation by Ninja. Each `rule` may have only one `command` declaration. To specify multiple commands use `&&` (or similar) to - concatenate operations. + concatenate operations. `depfile`:: path to an optional `Makefile` that contains extra _implicit dependencies_ (see <<ref_dependencies,the reference on @@ -683,6 +724,20 @@ aborting due to a missing input. rebuilt if the command line changes; and secondly, they are not cleaned by default. +`in`:: the shell-quoted space-separated list of files provided as + inputs to the build line referencing this `rule`. (`$in` is provided + solely for convenience; if you need some subset or variant of this + list of files, just construct a new variable with that list and use + that instead.) + +`in_newline`:: the same as `$in` except that multiple inputs are + separated by newlines rather than spaces. (For use with + `$rspfile_content`; this works around a bug in the MSVC linker where + it uses a fixed-size buffer for processing input.) + +`out`:: the shell-quoted space-separated list of files provided as + outputs to the build line referencing this `rule`. + `restat`:: if present, causes Ninja to re-stat the command's outputs after execution of the command. Each output whose modification time the command did not change will be treated as though it had never @@ -709,13 +764,9 @@ rule link build myapp.exe: link a.obj b.obj [possibly many other .obj files] ---- -Finally, the special `$in` and `$out` variables expand to the -shell-quoted space-separated list of files provided to the `build` -line referencing this `rule`. - +[[ref_dependencies]] Build dependencies ~~~~~~~~~~~~~~~~~~ -[[ref_dependencies]] There are three types of build dependencies which are subtly different. @@ -752,9 +803,34 @@ header file before starting a subsequent compilation step. (Once the header is used in compilation, a generated dependency file will then express the implicit dependency.) +Variable expansion +~~~~~~~~~~~~~~~~~~ + +Variables are expanded in paths (in a `build` or `default` statement) +and on the right side of a `name = value` statement. + +When a `name = value` statement is evaluated, its right-hand side is +expanded immediately (according to the below scoping rules), and +from then on `$name` expands to the static string as the result of the +expansion. It is never the case that you'll need to "double-escape" a +value to prevent it from getting expanded twice. + +All variables are expanded immediately as they're encountered in parsing, +with one important exception: variables in `rule` blocks are expanded +when the rule is _used_, not when it is declared. In the following +example, the `demo` rule prints "this is a demo of bar". + +---- +rule demo + command = echo "this is a demo of $foo' + +build out: demo + foo = bar +---- + +[[ref_scope]] Evaluation and scoping ~~~~~~~~~~~~~~~~~~~~~~ -[[ref_scope]] Top-level variable declarations are scoped to the file they occur in. @@ -767,26 +843,19 @@ To include another `.ninja` file in the current scope, much like a C `#include` statement, use `include` instead of `subninja`. Variable declarations indented in a `build` block are scoped to the -`build` block. This scope is inherited by the `rule`. The full -lookup order for a variable referenced in a rule is: +`build` block. The full lookup order for a variable expanded in a +`build` block (or the `rule` is uses) is: -1. Rule-level variables (i.e. `$in`, `$command`). +1. Special built-in variables (`$in`, `$out`). -2. Build-level variables from the `build` that references this rule. +2. Build-level variables from the `build` block. -3. File-level variables from the file that the `build` line was in. +3. Rule-level variables from the `rule` block (i.e. `$command`). + (Note from the above discussion on expansion that these are + expanded "late", and may make use of in-scope bindings like `$in`.) -4. Variables from the file that included that file using the - `subninja` keyword. +4. File-level variables from the file that the `build` line was in. -Variable expansion -~~~~~~~~~~~~~~~~~~ - -Variables are expanded in paths (in a `build` or `default` statement) -and on the right side of a `name = value` statement. +5. Variables from the file that included that file using the + `subninja` keyword. -When a `name = value` statement is evaluated, its right-hand side is -expanded once (according to the above scoping rules) immediately, and -from then on `$name` expands to the static string as the result of the -expansion. It is never the case that you'll need to "double-escape" a -value to prevent it from getting expanded twice. diff --git a/doc/style.css b/doc/style.css new file mode 100644 index 0000000..fc22ec1 --- /dev/null +++ b/doc/style.css @@ -0,0 +1,29 @@ +body { + margin: 5ex 10ex; + max-width: 40em; + line-height: 1.4; + font-family: sans-serif; + font-size: 0.8em; +} +h1, h2, h3 { + font-weight: normal; +} +pre, code { + font-family: x, monospace; +} +pre { + padding: 1ex; + background: #eee; + border: solid 1px #ddd; + min-width: 0; + font-size: 90%; +} +code { + color: #007; +} +.chapter { + margin-top: 4em; +} +p { + margin-top: 0; +} diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec index 2f009f6..f0c46fe 100644 --- a/misc/packaging/ninja.spec +++ b/misc/packaging/ninja.spec @@ -5,8 +5,10 @@ Release: %{rel}%{?dist} Group: Development/Tools License: Apache 2.0 URL: https://github.com/martine/ninja -Source0: %{name}-%{version}-%{release}.tar.gz -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release} +Source0: %{name}-%{version}-%{rel}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{rel} + +BuildRequires: asciidoc %description Ninja is yet another build system. It takes as input the interdependencies of files (typically source code and output executables) and @@ -17,7 +19,7 @@ which has over 30,000 source files and whose other build systems (including one seconds to start building after changing one file. Ninja is under a second. %prep -%setup -q -n %{name}-%{version}-%{release} +%setup -q -n %{name}-%{version}-%{rel} %build echo Building.. diff --git a/src/browse.py b/src/browse.py index 7f15e50..652bac2 100755 --- a/src/browse.py +++ b/src/browse.py @@ -29,6 +29,7 @@ except ImportError: import subprocess import sys import webbrowser +import urllib2 from collections import namedtuple Node = namedtuple('Node', ['inputs', 'rule', 'target', 'outputs']) @@ -151,7 +152,7 @@ def ninja_dump(target): class RequestHandler(httpserver.BaseHTTPRequestHandler): def do_GET(self): assert self.path[0] == '/' - target = self.path[1:] + target = urllib2.unquote(self.path[1:]) if target == '': self.send_response(302) diff --git a/src/build.cc b/src/build.cc index b4229c4..ae47a50 100644 --- a/src/build.cc +++ b/src/build.cc @@ -238,6 +238,13 @@ string BuildStatus::FormatProgressStatus( out += buf; break; + case 'e': { + double elapsed = overall_rate_.Elapsed(); + snprintf(buf, sizeof(buf), "%.3f", elapsed); + out += buf; + break; + } + default: Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); return ""; @@ -256,9 +263,9 @@ void BuildStatus::PrintStatus(Edge* edge) { bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; - string to_print = edge->GetDescription(); + string to_print = edge->GetBinding("description"); if (to_print.empty() || force_full_command) - to_print = edge->EvaluateCommand(); + to_print = edge->GetBinding("command"); #ifdef _WIN32 CONSOLE_SCREEN_BUFFER_INFO csbi; @@ -418,6 +425,12 @@ Edge* Plan::FindWork() { void Plan::ScheduleWork(Edge* edge) { Pool* pool = edge->pool(); if (pool->ShouldDelayEdge()) { + // The graph is not completely clean. Some Nodes have duplicate Out edges. + // We need to explicitly ignore these here, otherwise their work will get + // scheduled twice (see https://github.com/martine/ninja/pull/519) + if (ready_.count(edge)) { + return; + } pool->DelayEdge(edge); pool->RetrieveReadyEdges(&ready_); } else { @@ -612,7 +625,7 @@ void Builder::Cleanup() { for (vector<Edge*>::iterator i = active_edges.begin(); i != active_edges.end(); ++i) { - bool has_depfile = !(*i)->rule_->depfile().empty(); + string depfile = (*i)->GetBinding("depfile"); for (vector<Node*>::iterator ni = (*i)->outputs_.begin(); ni != (*i)->outputs_.end(); ++ni) { // Only delete this output if it was actually modified. This is @@ -622,12 +635,13 @@ void Builder::Cleanup() { // need to rebuild an output because of a modified header file // mentioned in a depfile, and the command touches its depfile // but is interrupted before it touches its output file.) - if (has_depfile || - (*ni)->mtime() != disk_interface_->Stat((*ni)->path())) + if (!depfile.empty() || + (*ni)->mtime() != disk_interface_->Stat((*ni)->path())) { disk_interface_->RemoveFile((*ni)->path()); + } } - if (has_depfile) - disk_interface_->RemoveFile((*i)->EvaluateDepFile()); + if (!depfile.empty()) + disk_interface_->RemoveFile(depfile); } } } @@ -771,11 +785,11 @@ bool Builder::StartEdge(Edge* edge, string* err) { // Create response file, if needed // XXX: this may also block; do we care? - if (edge->HasRspFile()) { - if (!disk_interface_->WriteFile(edge->GetRspFile(), - edge->GetRspFileContent())) { + string rspfile = edge->GetBinding("rspfile"); + if (!rspfile.empty()) { + string content = edge->GetBinding("rspfile_content"); + if (!disk_interface_->WriteFile(rspfile, content)) return false; - } } // start command computing and run it @@ -792,7 +806,7 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { TimeStamp restat_mtime = 0; if (success) { - if (edge->rule().restat() && !config_.dry_run) { + if (edge->GetBindingBool("restat") && !config_.dry_run) { bool node_cleaned = false; for (vector<Node*>::iterator i = edge->outputs_.begin(); @@ -817,9 +831,9 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { restat_mtime = input_mtime; } - if (restat_mtime != 0 && !edge->rule().depfile().empty()) { - TimeStamp depfile_mtime = - disk_interface_->Stat(edge->EvaluateDepFile()); + string depfile = edge->GetBinding("depfile"); + if (restat_mtime != 0 && !depfile.empty()) { + TimeStamp depfile_mtime = disk_interface_->Stat(depfile); if (depfile_mtime > restat_mtime) restat_mtime = depfile_mtime; } @@ -830,9 +844,10 @@ void Builder::FinishEdge(Edge* edge, bool success, const string& output) { } } - // delete the response file on success (if exists) - if (edge->HasRspFile()) - disk_interface_->RemoveFile(edge->GetRspFile()); + // Delete the response file on success (if exists) + string rspfile = edge->GetBinding("rspfile"); + if (!rspfile.empty()) + disk_interface_->RemoveFile(rspfile); plan_.EdgeFinished(edge); } diff --git a/src/build.h b/src/build.h index 23f653e..5747170 100644 --- a/src/build.h +++ b/src/build.h @@ -220,6 +220,7 @@ struct BuildStatus { RateInfo() : rate_(-1) {} void Restart() { stopwatch_.Restart(); } + double Elapsed() const { return stopwatch_.Elapsed(); } double rate() { return rate_; } void UpdateRate(int edges) { diff --git a/src/build_test.cc b/src/build_test.cc index 59c4c53..40a82ed 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -302,6 +302,80 @@ TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_FALSE(plan_.FindWork()); } +TEST_F(PlanTest, PoolWithRedundantEdges) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "pool compile\n" + " depth = 1\n" + "rule gen_foo\n" + " command = touch foo.cpp\n" + "rule gen_bar\n" + " command = touch bar.cpp\n" + "rule echo\n" + " command = echo $out > $out\n" + "build foo.cpp.obj: echo foo.cpp || foo.cpp\n" + " pool = compile\n" + "build bar.cpp.obj: echo bar.cpp || bar.cpp\n" + " pool = compile\n" + "build libfoo.a: echo foo.cpp.obj bar.cpp.obj\n" + "build foo.cpp: gen_foo\n" + "build bar.cpp: gen_bar\n" + "build all: phony libfoo.a\n")); + GetNode("foo.cpp")->MarkDirty(); + GetNode("foo.cpp.obj")->MarkDirty(); + GetNode("bar.cpp")->MarkDirty(); + GetNode("bar.cpp.obj")->MarkDirty(); + GetNode("libfoo.a")->MarkDirty(); + GetNode("all")->MarkDirty(); + string err; + EXPECT_TRUE(plan_.AddTarget(GetNode("all"), &err)); + ASSERT_EQ("", err); + ASSERT_TRUE(plan_.more_to_do()); + + Edge* edge = NULL; + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("foo.cpp", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("foo.cpp", edge->inputs_[0]->path()); + ASSERT_EQ("foo.cpp", edge->inputs_[1]->path()); + ASSERT_EQ("foo.cpp.obj", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("bar.cpp", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("bar.cpp", edge->inputs_[0]->path()); + ASSERT_EQ("bar.cpp", edge->inputs_[1]->path()); + ASSERT_EQ("bar.cpp.obj", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("foo.cpp.obj", edge->inputs_[0]->path()); + ASSERT_EQ("bar.cpp.obj", edge->inputs_[1]->path()); + ASSERT_EQ("libfoo.a", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_TRUE(edge); + ASSERT_EQ("libfoo.a", edge->inputs_[0]->path()); + ASSERT_EQ("all", edge->outputs_[0]->path()); + plan_.EdgeFinished(edge); + + edge = plan_.FindWork(); + ASSERT_FALSE(edge); + ASSERT_FALSE(plan_.more_to_do()); +} + + struct BuildTest : public StateTestWithBuiltinRules, public CommandRunner { BuildTest() : config_(MakeConfig()), diff --git a/src/clean.cc b/src/clean.cc index 0b8476b..12afb98 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -83,11 +83,11 @@ bool Cleaner::IsAlreadyRemoved(const string& path) { } void Cleaner::RemoveEdgeFiles(Edge* edge) { - string depfile = edge->EvaluateDepFile(); + string depfile = edge->GetBinding("depfile"); if (!depfile.empty()) Remove(depfile); - string rspfile = edge->GetRspFile(); + string rspfile = edge->GetBinding("rspfile"); if (!rspfile.empty()) Remove(rspfile); } @@ -117,7 +117,7 @@ int Cleaner::CleanAll(bool generator) { if ((*e)->is_phony()) continue; // Do not remove generator's files unless generator specified. - if (!generator && (*e)->rule().generator()) + if (!generator && (*e)->GetBindingBool("generator")) continue; for (vector<Node*>::iterator out_node = (*e)->outputs_.begin(); out_node != (*e)->outputs_.end(); ++out_node) { diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 6887c91..5a30c6b 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -53,10 +53,10 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 128, 128, 0, 128, 128, 128, 128, 128, + 0, 128, 128, 128, 128, 128, 128, 128, + 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 128, 0, 0, + 128, 128, 128, 128, 128, 128, 0, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -84,56 +84,48 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; - if (yych <= 'Z') { - if (yych <= '*') { - if (yych <= 0x00) goto yy6; - if (yych <= '\'') goto yy8; - if (yych <= ')') goto yy4; - goto yy8; + if (yych <= '[') { + if (yych <= '$') { + if (yych <= 0x00) goto yy7; + if (yych <= ' ') goto yy9; + if (yych <= '#') goto yy6; + goto yy4; } else { - if (yych <= '<') { - if (yych <= ':') goto yy4; - goto yy8; - } else { - if (yych <= '=') goto yy4; - if (yych <= '?') goto yy8; - goto yy4; - } + if (yych <= '=') goto yy6; + if (yych <= '?') goto yy9; + if (yych <= 'Z') goto yy6; + goto yy9; } } else { - if (yych <= '_') { - if (yych == '\\') goto yy2; - if (yych <= '^') goto yy8; - goto yy4; + if (yych <= '`') { + if (yych <= '\\') goto yy2; + if (yych == '_') goto yy6; + goto yy9; } else { - if (yych <= 'z') { - if (yych <= '`') goto yy8; - goto yy4; - } else { - if (yych == '~') goto yy4; - goto yy8; - } + if (yych <= 'z') goto yy6; + if (yych == '~') goto yy6; + goto yy9; } } yy2: ++in; - if ((yych = *in) <= '$') { + if ((yych = *in) <= '#') { if (yych <= '\n') { if (yych <= 0x00) goto yy3; - if (yych <= '\t') goto yy11; + if (yych <= '\t') goto yy14; } else { - if (yych == ' ') goto yy13; - if (yych <= '"') goto yy11; - goto yy13; + if (yych == ' ') goto yy16; + if (yych <= '"') goto yy14; + goto yy16; } } else { if (yych <= 'Z') { - if (yych == '*') goto yy13; - goto yy11; + if (yych == '*') goto yy16; + goto yy14; } else { - if (yych <= '\\') goto yy13; - if (yych == '|') goto yy13; - goto yy11; + if (yych <= '\\') goto yy16; + if (yych == '|') goto yy16; + goto yy14; } } yy3: @@ -144,8 +136,8 @@ yy3: } yy4: ++in; - yych = *in; - goto yy10; + if ((yych = *in) == '$') goto yy12; + goto yy11; yy5: { // Got a span of plain text. @@ -157,22 +149,35 @@ yy5: continue; } yy6: + yych = *++in; + goto yy11; +yy7: ++in; { break; } -yy8: +yy9: yych = *++in; goto yy3; -yy9: +yy10: ++in; yych = *in; -yy10: +yy11: if (yybm[0+yych] & 128) { - goto yy9; + goto yy10; } goto yy5; -yy11: +yy12: + ++in; + if (yybm[0+(yych = *in)] & 128) { + goto yy10; + } + { + // De-escape dollar character. + *out++ = '$'; + continue; + } +yy14: ++in; { // Let backslash before other characters through verbatim. @@ -180,7 +185,7 @@ yy11: *out++ = yych; continue; } -yy13: +yy16: ++in; { // De-escape backslashed character. diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 1d4a177..cf24a09 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -55,20 +55,25 @@ bool DepfileParser::Parse(string* content, string* err) { re2c:indent:string = " "; nul = "\000"; - escape = [ \\#*$[|]; + escape = [ \\#*[|]; '\\' escape { // De-escape backslashed character. *out++ = yych; continue; } + '$$' { + // De-escape dollar character. + *out++ = '$'; + continue; + } '\\' [^\000\n] { // Let backslash before other characters through verbatim. *out++ = '\\'; *out++ = yych; continue; } - [a-zA-Z0-9+,/_:.~()@=-]+ { + [a-zA-Z0-9+,/_:.~()@=-!]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 93d42db..0f6771a 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -95,7 +95,7 @@ TEST_F(DepfileParserTest, Escapes) { // it through. string err; EXPECT_TRUE(Parse( -"\\!\\@\\#\\$\\%\\^\\&\\\\", +"\\!\\@\\#$$\\%\\^\\&\\\\", &err)); ASSERT_EQ("", err); EXPECT_EQ("\\!\\@#$\\%\\^\\&\\", @@ -104,10 +104,12 @@ TEST_F(DepfileParserTest, Escapes) { } TEST_F(DepfileParserTest, SpecialChars) { + // See filenames like istreambuf.iterator_op!= in + // https://github.com/google/libcxx/tree/master/test/iterators/stream.iterators/istreambuf.iterator/ string err; EXPECT_TRUE(Parse( "C:/Program\\ Files\\ (x86)/Microsoft\\ crtdefs.h: \n" -" en@quot.header~ t+t-x=1", +" en@quot.header~ t+t-x!=1", &err)); ASSERT_EQ("", err); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", @@ -115,7 +117,7 @@ TEST_F(DepfileParserTest, SpecialChars) { ASSERT_EQ(2u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); - EXPECT_EQ("t+t-x=1", + EXPECT_EQ("t+t-x!=1", parser_.ins_[1].AsString()); } diff --git a/src/eval_env.cc b/src/eval_env.cc index 81a8765..834b7e1 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -27,6 +27,22 @@ void BindingEnv::AddBinding(const string& key, const string& val) { bindings_[key] = val; } +string BindingEnv::LookupWithFallback(const string& var, + const EvalString* eval, + Env* env) { + map<string, string>::iterator i = bindings_.find(var); + if (i != bindings_.end()) + return i->second; + + if (eval) + return eval->Evaluate(env); + + if (parent_) + return parent_->LookupVariable(var); + + return ""; +} + string EvalString::Evaluate(Env* env) const { string result; for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) { diff --git a/src/eval_env.h b/src/eval_env.h index 6e0a0c0..f3c959a 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -22,6 +22,8 @@ using namespace std; #include "string_piece.h" +struct EvalString; + /// An interface for a scope for variable (e.g. "$foo") lookups. struct Env { virtual ~Env() {} @@ -33,10 +35,20 @@ struct Env { struct BindingEnv : public Env { BindingEnv() : parent_(NULL) {} explicit BindingEnv(Env* parent) : parent_(parent) {} + virtual ~BindingEnv() {} virtual string LookupVariable(const string& var); + void AddBinding(const string& key, const string& val); + /// This is tricky. Edges want lookup scope to go in this order: + /// 1) value set on edge itself (edge_->env_) + /// 2) value set on rule, with expansion in the edge's scope + /// 3) value set on enclosing scope of edge (edge_->env_->parent_) + /// This function takes as parameters the necessary info to do (2). + string LookupWithFallback(const string& var, const EvalString* eval, + Env* env); + private: map<string, string> bindings_; Env* parent_; diff --git a/src/graph.cc b/src/graph.cc index f9b9c6f..b000c48 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -32,16 +32,40 @@ bool Node::Stat(DiskInterface* disk_interface) { return mtime_ > 0; } +void Rule::AddBinding(const string& key, const EvalString& val) { + bindings_[key] = val; +} + +const EvalString* Rule::GetBinding(const string& key) const { + map<string, EvalString>::const_iterator i = bindings_.find(key); + if (i == bindings_.end()) + return NULL; + return &i->second; +} + +// static +bool Rule::IsReservedBinding(const string& var) { + return var == "command" || + var == "depfile" || + var == "description" || + var == "generator" || + var == "pool" || + var == "restat" || + var == "rspfile" || + var == "rspfile_content"; +} + bool DependencyScan::RecomputeDirty(Edge* edge, string* err) { bool dirty = false; edge->outputs_ready_ = true; - if (!edge->rule_->depfile().empty()) { - if (!LoadDepFile(edge, err)) { + string depfile = edge->GetBinding("depfile"); + if (!depfile.empty()) { + if (!LoadDepFile(edge, depfile, err)) { if (!err->empty()) return false; EXPLAIN("Edge targets are dirty because depfile '%s' is missing", - edge->EvaluateDepFile().c_str()); + depfile.c_str()); dirty = true; } } @@ -142,7 +166,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // build log. Use that mtime instead, so that the file will only be // considered dirty if an input was modified since the previous run. TimeStamp most_recent_stamp = most_recent_input->mtime(); - if (edge->rule_->restat() && build_log() && + if (edge->GetBindingBool("restat") && build_log() && (entry = build_log()->LookupByOutput(output->path()))) { if (entry->restat_mtime < most_recent_stamp) { EXPLAIN("restat of output %s older than most recent input %s " @@ -162,7 +186,7 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - if (!edge->rule_->generator() && build_log()) { + if (!edge->GetBindingBool("generator") && build_log()) { if (entry || (entry = build_log()->LookupByOutput(output->path()))) { if (BuildLog::LogEntry::HashCommand(command) != entry->command_hash) { EXPLAIN("command line changed for %s", output->path().c_str()); @@ -212,11 +236,11 @@ string EdgeEnv::LookupVariable(const string& var) { return MakePathList(edge_->outputs_.begin(), edge_->outputs_.end(), ' '); - } else if (edge_->env_) { - return edge_->env_->LookupVariable(var); - } else { - return string(); } + + // See notes on BindingEnv::LookupWithFallback. + const EvalString* eval = edge_->rule_->GetBinding(var); + return edge_->env_->LookupWithFallback(var, eval, this); } string EdgeEnv::MakePathList(vector<Node*>::iterator begin, @@ -239,43 +263,31 @@ string EdgeEnv::MakePathList(vector<Node*>::iterator begin, } string Edge::EvaluateCommand(bool incl_rsp_file) { - EdgeEnv env(this); - string command = rule_->command().Evaluate(&env); - if (incl_rsp_file && HasRspFile()) - command += ";rspfile=" + GetRspFileContent(); + string command = GetBinding("command"); + if (incl_rsp_file) { + string rspfile_content = GetBinding("rspfile_content"); + if (!rspfile_content.empty()) + command += ";rspfile=" + rspfile_content; + } return command; } -string Edge::EvaluateDepFile() { +string Edge::GetBinding(const string& key) { EdgeEnv env(this); - return rule_->depfile().Evaluate(&env); + return env.LookupVariable(key); } -string Edge::GetDescription() { - EdgeEnv env(this); - return rule_->description().Evaluate(&env); -} - -bool Edge::HasRspFile() { - return !rule_->rspfile().empty(); -} - -string Edge::GetRspFile() { - EdgeEnv env(this); - return rule_->rspfile().Evaluate(&env); +bool Edge::GetBindingBool(const string& key) { + return !GetBinding(key).empty(); } -string Edge::GetRspFileContent() { - EdgeEnv env(this); - return rule_->rspfile_content().Evaluate(&env); -} - -bool DependencyScan::LoadDepFile(Edge* edge, string* err) { +bool DependencyScan::LoadDepFile(Edge* edge, const string& path, string* err) { METRIC_RECORD("depfile load"); - string path = edge->EvaluateDepFile(); string content = disk_interface_->ReadFile(path, err); - if (!err->empty()) + if (!err->empty()) { + *err = "loading '" + path + "': " + *err; return false; + } // On a missing depfile: return false and empty *err. if (content.empty()) return false; @@ -317,8 +329,7 @@ bool DependencyScan::LoadDepFile(Edge* edge, string* err) { // create one; this makes us not abort if the input is missing, // but instead will rebuild in that circumstance. if (!node->in_edge()) { - Edge* phony_edge = state_->AddEdge(&State::kPhonyRule, - &State::kDefaultPool); + Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); node->set_in_edge(phony_edge); phony_edge->outputs_.push_back(node); diff --git a/src/graph.h b/src/graph.h index 3c31e19..8b93e29 100644 --- a/src/graph.h +++ b/src/graph.h @@ -102,38 +102,23 @@ private: /// An invokable build command and associated metadata (description, etc.). struct Rule { - explicit Rule(const string& name) - : name_(name), generator_(false), restat_(false) {} + explicit Rule(const string& name) : name_(name) {} const string& name() const { return name_; } - bool generator() const { return generator_; } - bool restat() const { return restat_; } + typedef map<string, EvalString> Bindings; + void AddBinding(const string& key, const EvalString& val); - const EvalString& command() const { return command_; } - const EvalString& description() const { return description_; } - const EvalString& depfile() const { return depfile_; } - const EvalString& rspfile() const { return rspfile_; } - const EvalString& rspfile_content() const { return rspfile_content_; } + static bool IsReservedBinding(const string& var); - /// Used by a test. - void set_command(const EvalString& command) { command_ = command; } + const EvalString* GetBinding(const string& key) const; private: // Allow the parsers to reach into this object and fill out its fields. friend struct ManifestParser; string name_; - - bool generator_; - bool restat_; - - EvalString command_; - EvalString description_; - EvalString depfile_; - EvalString pool_; - EvalString rspfile_; - EvalString rspfile_content_; + map<string, EvalString> bindings_; }; struct BuildLog; @@ -153,17 +138,9 @@ struct Edge { /// If incl_rsp_file is enabled, the string will also contain the /// full contents of a response file (if applicable) string EvaluateCommand(bool incl_rsp_file = false); - string EvaluateDepFile(); - string GetDescription(); - - /// Does the edge use a response file? - bool HasRspFile(); - - /// Get the path to the response file - string GetRspFile(); - /// Get the contents of the response file - string GetRspFileContent(); + string GetBinding(const string& key); + bool GetBindingBool(const string& key); void Dump(const char* prefix="") const; @@ -171,7 +148,7 @@ struct Edge { Pool* pool_; vector<Node*> inputs_; vector<Node*> outputs_; - Env* env_; + BindingEnv* env_; bool outputs_ready_; const Rule& rule() const { return *rule_; } @@ -220,7 +197,7 @@ struct DependencyScan { bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input, const string& command, Node* output); - bool LoadDepFile(Edge* edge, string* err); + bool LoadDepFile(Edge* edge, const string& path, string* err); BuildLog* build_log() const { return build_log_; diff --git a/src/graph_test.cc b/src/graph_test.cc index 5b25c2f..396def4 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -187,3 +187,38 @@ TEST_F(GraphTest, DepfileRemoved) { ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.o")->dirty()); } + +// Check that rule-level variables are in scope for eval. +TEST_F(GraphTest, RuleVariablesInScope) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" depfile = x\n" +" command = depfile is $depfile\n" +"build out: r in\n")); + Edge* edge = GetNode("out")->in_edge(); + EXPECT_EQ("depfile is x", edge->EvaluateCommand()); +} + +// Check that build statements can override rule builtins like depfile. +TEST_F(GraphTest, DepfileOverride) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" depfile = x\n" +" command = unused\n" +"build out: r in\n" +" depfile = y\n")); + Edge* edge = GetNode("out")->in_edge(); + EXPECT_EQ("y", edge->GetBinding("depfile")); +} + +// Check that overridden values show up in expansion of rule-level bindings. +TEST_F(GraphTest, DepfileOverrideParent) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" depfile = x\n" +" command = depfile is $depfile\n" +"build out: r in\n" +" depfile = y\n")); + Edge* edge = GetNode("out")->in_edge(); + EXPECT_EQ("depfile is y", edge->GetBinding("command")); +} diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 2d052b5..14fca73 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -23,6 +23,7 @@ #include "metrics.h" #include "state.h" #include "util.h" +#include "version.h" ManifestParser::ManifestParser(State* state, FileReader* file_reader) : state_(state), file_reader_(file_reader) { @@ -66,10 +67,15 @@ bool ManifestParser::Parse(const string& filename, const string& input, case Lexer::IDENT: { lexer_.UnreadToken(); string name; - EvalString value; - if (!ParseLet(&name, &value, err)) + EvalString let_value; + if (!ParseLet(&name, &let_value, err)) return false; - env_->AddBinding(name, value.Evaluate(env_)); + string value = let_value.Evaluate(env_); + // Check ninja_required_version immediately so we can exit + // before encountering any syntactic surprises. + if (name == "ninja_required_version") + CheckNinjaVersion(value); + env_->AddBinding(name, value); break; } case Lexer::INCLUDE: @@ -154,22 +160,8 @@ bool ManifestParser::ParseRule(string* err) { if (!ParseLet(&key, &value, err)) return false; - if (key == "command") { - rule->command_ = value; - } else if (key == "depfile") { - rule->depfile_ = value; - } else if (key == "description") { - rule->description_ = value; - } else if (key == "generator") { - rule->generator_ = true; - } else if (key == "restat") { - rule->restat_ = true; - } else if (key == "rspfile") { - rule->rspfile_ = value; - } else if (key == "rspfile_content") { - rule->rspfile_content_ = value; - } else if (key == "pool") { - rule->pool_ = value; + if (Rule::IsReservedBinding(key)) { + rule->AddBinding(key, value); } else { // Die on other keyvals for now; revisit if we want to add a // scope here. @@ -177,12 +169,13 @@ bool ManifestParser::ParseRule(string* err) { } } - if (rule->rspfile_.empty() != rule->rspfile_content_.empty()) { - return lexer_.Error("rspfile and rspfile_content need to be both specified", - err); + if (rule->bindings_["rspfile"].empty() != + rule->bindings_["rspfile_content"].empty()) { + return lexer_.Error("rspfile and rspfile_content need to be " + "both specified", err); } - if (rule->command_.empty()) + if (rule->bindings_["command"].empty()) return lexer_.Error("expected 'command =' line", err); state_->AddRule(rule); @@ -296,42 +289,29 @@ bool ManifestParser::ParseEdge(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - // Default to using outer env. - BindingEnv* env = env_; - Pool* pool = NULL; + // XXX scoped_ptr to handle error case. + BindingEnv* env = new BindingEnv(env_); - // But create and fill a nested env if there are variables in scope. - if (lexer_.PeekToken(Lexer::INDENT)) { - // XXX scoped_ptr to handle error case. - env = new BindingEnv(env_); - do { - string key; - EvalString val; - if (!ParseLet(&key, &val, err)) - return false; - if (key == "pool") { - string pool_name = val.Evaluate(env_); - pool = state_->LookupPool(pool_name); - if (pool == NULL) - return lexer_.Error("undefined pool '" + pool_name + "'", err); - } else { - env->AddBinding(key, val.Evaluate(env_)); - } - } while (lexer_.PeekToken(Lexer::INDENT)); - } + while (lexer_.PeekToken(Lexer::INDENT)) { + string key; + EvalString val; + if (!ParseLet(&key, &val, err)) + return false; - if (pool == NULL) { - if (!rule->pool_.empty()) { - pool = state_->LookupPool(rule->pool_.Evaluate(env_)); - if (pool == NULL) - return lexer_.Error("cannot resolve pool for this edge.", err); - } else { - pool = &State::kDefaultPool; - } + env->AddBinding(key, val.Evaluate(env_)); } - Edge* edge = state_->AddEdge(rule, pool); + Edge* edge = state_->AddEdge(rule); edge->env_ = env; + + string pool_name = edge->GetBinding("pool"); + if (!pool_name.empty()) { + Pool* pool = state_->LookupPool(pool_name); + if (pool == NULL) + return lexer_.Error("unknown pool name", err); + edge->pool_ = pool; + } + for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) { string path = i->Evaluate(env); string path_err; diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index 8b00efb..4ac093f 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -61,7 +61,8 @@ TEST_F(ParserTest, Rules) { ASSERT_EQ(3u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("cat", rule->name()); - EXPECT_EQ("[cat ][$in][ > ][$out]", rule->command().Serialize()); + EXPECT_EQ("[cat ][$in][ > ][$out]", + rule->GetBinding("command")->Serialize()); } TEST_F(ParserTest, RuleAttributes) { @@ -92,8 +93,9 @@ TEST_F(ParserTest, IgnoreIndentedComments) { ASSERT_EQ(2u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("cat", rule->name()); - EXPECT_TRUE(rule->restat()); - EXPECT_FALSE(rule->generator()); + Edge* edge = state.GetNode("result")->in_edge(); + EXPECT_TRUE(edge->GetBindingBool("restat")); + EXPECT_FALSE(edge->GetBindingBool("generator")); } TEST_F(ParserTest, IgnoreIndentedBlankLines) { @@ -124,9 +126,10 @@ TEST_F(ParserTest, ResponseFiles) { ASSERT_EQ(2u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("cat_rsp", rule->name()); - EXPECT_EQ("[cat ][$rspfile][ > ][$out]", rule->command().Serialize()); - EXPECT_EQ("[$rspfile]", rule->rspfile().Serialize()); - EXPECT_EQ("[$in]", rule->rspfile_content().Serialize()); + EXPECT_EQ("[cat ][$rspfile][ > ][$out]", + rule->GetBinding("command")->Serialize()); + EXPECT_EQ("[$rspfile]", rule->GetBinding("rspfile")->Serialize()); + EXPECT_EQ("[$in]", rule->GetBinding("rspfile_content")->Serialize()); } TEST_F(ParserTest, InNewline) { @@ -140,7 +143,8 @@ TEST_F(ParserTest, InNewline) { ASSERT_EQ(2u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("cat_rsp", rule->name()); - EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->command().Serialize()); + EXPECT_EQ("[cat ][$in_newline][ > ][$out]", + rule->GetBinding("command")->Serialize()); Edge* edge = state.edges_[0]; EXPECT_EQ("cat in\nin2 > out", edge->EvaluateCommand()); @@ -200,7 +204,7 @@ TEST_F(ParserTest, Continuation) { ASSERT_EQ(2u, state.rules_.size()); const Rule* rule = state.rules_.begin()->second; EXPECT_EQ("link", rule->name()); - EXPECT_EQ("[foo bar baz]", rule->command().Serialize()); + EXPECT_EQ("[foo bar baz]", rule->GetBinding("command")->Serialize()); } TEST_F(ParserTest, Backslash) { @@ -291,7 +295,8 @@ TEST_F(ParserTest, ReservedWords) { TEST_F(ParserTest, Errors) { { - ManifestParser parser(NULL, NULL); + State state; + ManifestParser parser(&state, NULL); string err; EXPECT_FALSE(parser.ParseTest("foobar", &err)); EXPECT_EQ("input:1: expected '=', got eof\n" @@ -301,7 +306,8 @@ TEST_F(ParserTest, Errors) { } { - ManifestParser parser(NULL, NULL); + State state; + ManifestParser parser(&state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x 3", &err)); EXPECT_EQ("input:1: expected '=', got identifier\n" @@ -311,7 +317,8 @@ TEST_F(ParserTest, Errors) { } { - ManifestParser parser(NULL, NULL); + State state; + ManifestParser parser(&state, NULL); string err; EXPECT_FALSE(parser.ParseTest("x = 3", &err)); EXPECT_EQ("input:1: unexpected EOF\n" diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 152450e..0bbe98b 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -44,6 +44,31 @@ void PushPathIntoEnvironment(const string& env_block) { } } +void WriteDepFileOrDie(const char* object_path, CLWrapper* cl) { + string depfile_path = string(object_path) + ".d"; + FILE* depfile = fopen(depfile_path.c_str(), "w"); + if (!depfile) { + unlink(object_path); + Fatal("opening %s: %s", depfile_path.c_str(), GetLastErrorString().c_str()); + } + if (fprintf(depfile, "%s: ", object_path) < 0) { + unlink(object_path); + fclose(depfile); + unlink(depfile_path.c_str()); + Fatal("writing %s", depfile_path.c_str()); + } + vector<string> headers = cl->GetEscapedResult(); + for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) { + if (fprintf(depfile, "%s\n", i->c_str()) < 0) { + unlink(object_path); + fclose(depfile); + unlink(depfile_path.c_str()); + Fatal("writing %s", depfile_path.c_str()); + } + } + fclose(depfile); +} + } // anonymous namespace int MSVCHelperMain(int argc, char** argv) { @@ -95,17 +120,7 @@ int MSVCHelperMain(int argc, char** argv) { cl.SetEnvBlock((void*)env.data()); int exit_code = cl.Run(command); - string depfile = string(output_filename) + ".d"; - FILE* output = fopen(depfile.c_str(), "w"); - if (!output) { - Fatal("opening %s: %s", depfile.c_str(), GetLastErrorString().c_str()); - } - fprintf(output, "%s: ", output_filename); - vector<string> headers = cl.GetEscapedResult(); - for (vector<string>::iterator i = headers.begin(); i != headers.end(); ++i) { - fprintf(output, "%s\n", i->c_str()); - } - fclose(output); + WriteDepFileOrDie(output_filename, &cl); return exit_code; } diff --git a/src/ninja.cc b/src/ninja.cc index 08d4b14..9529a50 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -15,6 +15,7 @@ #include <errno.h> #include <limits.h> #include <stdio.h> +#include <stdlib.h> #include <string.h> #include <sys/stat.h> #include <sys/types.h> @@ -41,6 +42,7 @@ #include "metrics.h" #include "state.h" #include "util.h" +#include "version.h" // Defined in msvc_helper_main-win32.cc. int MSVCHelperMain(int argc, char** argv); @@ -49,7 +51,7 @@ namespace { /// The version number of the current Ninja release. This will always /// be "git" on trunk. -const char* kVersion = "1.1.0"; +const char* kVersion = "1.2.0"; /// Global information passed into subtools. struct Globals { @@ -109,7 +111,7 @@ void Usage(const BuildConfig& config) { " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" "\n" -" -j N run N jobs in parallel [default=%d]\n" +" -j N run N jobs in parallel [default=%d, derived from CPUs available]\n" " -l N do not start new jobs if the load average is greater than N\n" #ifdef _WIN32 " (not yet implemented on Windows)\n" @@ -121,7 +123,7 @@ void Usage(const BuildConfig& config) { " -d MODE enable debugging (use -d list to list modes)\n" " -t TOOL run a subtool (use -t list to list subtools)\n" " terminates toplevel options; further flags are passed to the tool\n", - kVersion, config.parallelism); + kNinjaVersion, config.parallelism); } /// Choose a default value for the -j (parallelism) flag. @@ -410,23 +412,6 @@ int ToolTargets(Globals* globals, int argc, char* argv[]) { } } -int ToolRules(Globals* globals, int argc, char* /* argv */[]) { - for (map<string, const Rule*>::iterator i = globals->state->rules_.begin(); - i != globals->state->rules_.end(); ++i) { - if (i->second->description().empty()) { - printf("%s\n", i->first.c_str()); - } else { - printf("%s: %s\n", - i->first.c_str(), - // XXX I changed it such that we don't have an easy way - // to get the source text anymore, so this output is - // unsatisfactory. How useful is this command, anyway? - i->second->description().Serialize().c_str()); - } - } - return 0; -} - void PrintCommands(Edge* edge, set<Edge*>* seen) { if (!edge) return; @@ -505,6 +490,49 @@ int ToolClean(Globals* globals, int argc, char* argv[]) { } } +void EncodeJSONString(const char *str) { + while (*str) { + if (*str == '"' || *str == '\\') + putchar('\\'); + putchar(*str); + str++; + } +} + +int ToolCompilationDatabase(Globals* globals, int argc, char* argv[]) { + bool first = true; + char cwd[PATH_MAX]; + + if (!getcwd(cwd, PATH_MAX)) { + Error("cannot determine working directory: %s", strerror(errno)); + return 1; + } + + putchar('['); + for (vector<Edge*>::iterator e = globals->state->edges_.begin(); + e != globals->state->edges_.end(); ++e) { + for (int i = 0; i != argc; ++i) { + if ((*e)->rule_->name() == argv[i]) { + if (!first) + putchar(','); + + printf("\n {\n \"directory\": \""); + EncodeJSONString(cwd); + printf("\",\n \"command\": \""); + EncodeJSONString((*e)->EvaluateCommand().c_str()); + printf("\",\n \"file\": \""); + EncodeJSONString((*e)->inputs_[0]->path().c_str()); + printf("\"\n }"); + + first = false; + } + } + } + + puts("\n]"); + return 0; +} + int ToolUrtle(Globals* globals, int argc, char** argv) { // RLE encoded. const char* urtle = @@ -551,10 +579,10 @@ int ChooseTool(const string& tool_name, const Tool** tool_out) { Tool::RUN_AFTER_LOAD, ToolGraph }, { "query", "show inputs/outputs for a path", Tool::RUN_AFTER_LOAD, ToolQuery }, - { "rules", "list all rules", - Tool::RUN_AFTER_LOAD, ToolRules }, { "targets", "list targets by their rule or depth in the DAG", Tool::RUN_AFTER_LOAD, ToolTargets }, + { "compdb", "dump JSON compilation database to stdout", + Tool::RUN_AFTER_LOAD, ToolCompilationDatabase }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, ToolUrtle }, { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } @@ -605,7 +633,14 @@ bool DebugEnable(const string& name, Globals* globals) { g_explaining = true; return true; } else { - printf("ninja: unknown debug setting '%s'\n", name.c_str()); + const char* suggestion = + SpellcheckString(name, "stats", "explain", NULL); + if (suggestion) { + Error("unknown debug setting '%s', did you mean '%s'?", + name.c_str(), suggestion); + } else { + Error("unknown debug setting '%s'", name.c_str()); + } return false; } } @@ -791,7 +826,7 @@ int NinjaMain(int argc, char** argv) { working_dir = optarg; break; case OPT_VERSION: - printf("%s\n", kVersion); + printf("%s\n", kNinjaVersion); return 0; case 'h': default: diff --git a/src/state.cc b/src/state.cc index bb0cc15..9f46fee 100644 --- a/src/state.cc +++ b/src/state.cc @@ -35,23 +35,25 @@ void Pool::EdgeFinished(const Edge& edge) { void Pool::DelayEdge(Edge* edge) { assert(depth_ != 0); - delayed_.push_back(edge); + delayed_.insert(edge); } void Pool::RetrieveReadyEdges(set<Edge*>* ready_queue) { - while (!delayed_.empty()) { - Edge* edge = delayed_.front(); + DelayedEdges::iterator it = delayed_.begin(); + while (it != delayed_.end()) { + Edge* edge = *it; if (current_use_ + edge->weight() > depth_) break; - delayed_.pop_front(); ready_queue->insert(edge); EdgeScheduled(*edge); + ++it; } + delayed_.erase(delayed_.begin(), it); } void Pool::Dump() const { printf("%s (%d/%d) ->\n", name_.c_str(), current_use_, depth_); - for (deque<Edge*>::const_iterator it = delayed_.begin(); + for (DelayedEdges::const_iterator it = delayed_.begin(); it != delayed_.end(); ++it) { printf("\t"); @@ -59,6 +61,13 @@ void Pool::Dump() const { } } +bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { + if (!a) return b; + if (!b) return false; + int weight_diff = a->weight() - b->weight(); + return ((weight_diff < 0) || (weight_diff == 0 && a < b)); +} + Pool State::kDefaultPool("", 0); const Rule State::kPhonyRule("phony"); @@ -91,10 +100,10 @@ Pool* State::LookupPool(const string& pool_name) { return i->second; } -Edge* State::AddEdge(const Rule* rule, Pool* pool) { +Edge* State::AddEdge(const Rule* rule) { Edge* edge = new Edge(); edge->rule_ = rule; - edge->pool_ = pool; + edge->pool_ = &State::kDefaultPool; edge->env_ = &bindings_; edges_.push_back(edge); return edge; diff --git a/src/state.h b/src/state.h index 918fe09..7e3aead 100644 --- a/src/state.h +++ b/src/state.h @@ -39,7 +39,7 @@ struct Rule; /// completes). struct Pool { explicit Pool(const string& name, int depth) - : name_(name), current_use_(0), depth_(depth) { } + : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) { } // A depth of 0 is infinite bool is_valid() const { return depth_ >= 0; } @@ -66,9 +66,7 @@ struct Pool { /// Dump the Pool and its edges (useful for debugging). void Dump() const; -private: - int UnitsWaiting() { return delayed_.size(); } - + private: string name_; /// |current_use_| is the total of the weights of the edges which are @@ -76,7 +74,10 @@ private: int current_use_; int depth_; - deque<Edge*> delayed_; + static bool WeightedEdgeCmp(const Edge* a, const Edge* b); + + typedef set<Edge*,bool(*)(const Edge*, const Edge*)> DelayedEdges; + DelayedEdges delayed_; }; /// Global state (file status, loaded rules) for a single run. @@ -92,7 +93,7 @@ struct State { void AddPool(Pool* pool); Pool* LookupPool(const string& pool_name); - Edge* AddEdge(const Rule* rule, Pool* pool); + Edge* AddEdge(const Rule* rule); Node* GetNode(StringPiece path); Node* LookupNode(StringPiece path); diff --git a/src/state_test.cc b/src/state_test.cc index 26177ff..af2bff1 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -29,10 +29,10 @@ TEST(State, Basic) { command.AddSpecial("out"); Rule* rule = new Rule("cat"); - rule->set_command(command); + rule->AddBinding("command", command); state.AddRule(rule); - Edge* edge = state.AddEdge(rule, &State::kDefaultPool); + Edge* edge = state.AddEdge(rule); state.AddIn(edge, "in1"); state.AddIn(edge, "in2"); state.AddOut(edge, "out"); diff --git a/src/util.cc b/src/util.cc index 4b2900f..91e8fad 100644 --- a/src/util.cc +++ b/src/util.cc @@ -39,7 +39,7 @@ #elif defined(__SVR4) && defined(__sun) #include <unistd.h> #include <sys/loadavg.h> -#elif defined(linux) +#elif defined(linux) || defined(__GLIBC__) #include <sys/sysinfo.h> #endif @@ -295,7 +295,7 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } -#if defined(linux) +#if defined(linux) || defined(__GLIBC__) int GetProcessorCount() { return get_nprocs(); } @@ -325,7 +325,7 @@ int GetProcessorCount() { } #endif -#ifdef _WIN32 +#if defined(_WIN32) || defined(__CYGWIN__) double GetLoadAverage() { // TODO(nicolas.despres@gmail.com): Find a way to implement it on Windows. // Remember to also update Usage() when this is fixed. @@ -76,6 +76,8 @@ string ElideMiddle(const string& str, size_t width); #define unlink _unlink #define chdir _chdir #define strtoull _strtoui64 +#define getcwd _getcwd +#define PATH_MAX _MAX_PATH #endif #ifdef _WIN32 diff --git a/src/version.cc b/src/version.cc new file mode 100644 index 0000000..b3a93d1 --- /dev/null +++ b/src/version.cc @@ -0,0 +1,57 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "version.h" + +#include <stdlib.h> + +#include "util.h" + +const char* kNinjaVersion = "1.1.0.git"; + +void ParseVersion(const string& version, int* major, int* minor) { + size_t end = version.find('.'); + *major = atoi(version.substr(0, end).c_str()); + *minor = 0; + if (end != string::npos) { + size_t start = end + 1; + end = version.find('.', start); + *minor = atoi(version.substr(start, end).c_str()); + } +} + +void CheckNinjaVersion(const string& version) { + int bin_major, bin_minor; + ParseVersion(kNinjaVersion, &bin_major, &bin_minor); + int file_major, file_minor; + ParseVersion(version, &file_major, &file_minor); + + if (bin_major > file_major) { + Warning("ninja executable version (%s) greater than build file " + "ninja_required_version (%s); versions may be incompatible.", + kNinjaVersion, version.c_str()); + return; + } + + if ((bin_major == file_major && bin_minor < file_minor) || + bin_major < file_major) { + Fatal("ninja version (%s) incompatible with build file " + "ninja_required_version version (%s).", + kNinjaVersion, version.c_str()); + } +} + + + + diff --git a/src/version.h b/src/version.h new file mode 100644 index 0000000..bd6b9ff --- /dev/null +++ b/src/version.h @@ -0,0 +1,32 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_VERSION_H_ +#define NINJA_VERSION_H_ + +#include <string> +using namespace std; + +/// The version number of the current Ninja release. This will always +/// be "git" on trunk. +extern const char* kNinjaVersion; + +/// Parse the major/minor components of a version string. +void ParseVersion(const string& version, int* major, int* minor); + +/// Check whether \a version is compatible with the current Ninja version, +/// aborting if not. +void CheckNinjaVersion(const string& required_version); + +#endif // NINJA_VERSION_H_ |