summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.clang-format25
-rw-r--r--.gitignore3
-rwxr-xr-xconfigure.py9
-rw-r--r--misc/bash-completion4
-rw-r--r--misc/ninja_syntax.py9
-rw-r--r--misc/write_fake_manifests.py219
-rw-r--r--src/depfile_parser_perftest.cc (renamed from src/parser_perftest.cc)0
-rw-r--r--src/disk_interface.cc16
-rw-r--r--src/disk_interface_test.cc13
-rw-r--r--src/manifest_parser.cc9
-rw-r--r--src/manifest_parser_perftest.cc112
-rw-r--r--src/msvc_helper-win32.cc2
-rw-r--r--src/msvc_helper_test.cc7
13 files changed, 406 insertions, 22 deletions
diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000..1841c03
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,25 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This isn't meant to be authoritative, but it's good enough to be useful.
+# Still use your best judgement for formatting decisions: clang-format
+# sometimes makes strange choices.
+
+BasedOnStyle: Google
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+Cpp11BracedListStyle: false
+IndentCaseLabels: false
diff --git a/.gitignore b/.gitignore
index 501a02d..40a610d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,9 +8,10 @@ TAGS
/ninja
/build_log_perftest
/canon_perftest
+/depfile_parser_perftest
/hash_collision_bench
/ninja_test
-/parser_perftest
+/manifest_parser_perftest
/graph.png
/doc/manual.html
/doc/doxygen
diff --git a/configure.py b/configure.py
index da2f6ef..c5a6abd 100755
--- a/configure.py
+++ b/configure.py
@@ -377,18 +377,21 @@ all_targets += ninja_test
n.comment('Ancillary executables.')
-objs = cxx('parser_perftest')
-all_targets += n.build(binary('parser_perftest'), 'link', objs,
- implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('build_log_perftest')
all_targets += n.build(binary('build_log_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('canon_perftest')
all_targets += n.build(binary('canon_perftest'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
+objs = cxx('depfile_parser_perftest')
+all_targets += n.build(binary('depfile_parser_perftest'), 'link', objs,
+ implicit=ninja_lib, variables=[('libs', libs)])
objs = cxx('hash_collision_bench')
all_targets += n.build(binary('hash_collision_bench'), 'link', objs,
implicit=ninja_lib, variables=[('libs', libs)])
+objs = cxx('manifest_parser_perftest')
+all_targets += n.build(binary('manifest_parser_perftest'), 'link', objs,
+ implicit=ninja_lib, variables=[('libs', libs)])
n.newline()
n.comment('Generate a graph using the "graph" tool.')
diff --git a/misc/bash-completion b/misc/bash-completion
index 2d6975b..719e7a8 100644
--- a/misc/bash-completion
+++ b/misc/bash-completion
@@ -26,12 +26,12 @@ _ninja_target() {
dir="."
line=$(echo ${COMP_LINE} | cut -d" " -f 2-)
# filter out all non relevant arguments but keep C for dirs
- while getopts C:f:j:l:k:nvd:t: opt "${line[@]}"; do
+ while getopts C:f:j:l:k:nvd:t: opt $line; do
case $opt in
C) dir="$OPTARG" ;;
esac
done;
- targets_command="ninja -C ${dir} -t targets all"
+ targets_command="eval ninja -C \"${dir}\" -t targets all"
targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}')
COMPREPLY=($(compgen -W "$targets" -- "$cur"))
fi
diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py
index d69e3e4..4b9b547 100644
--- a/misc/ninja_syntax.py
+++ b/misc/ninja_syntax.py
@@ -61,16 +61,15 @@ class Writer(object):
def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
variables=None):
outputs = self._as_list(outputs)
- all_inputs = self._as_list(inputs)[:]
- out_outputs = list(map(escape_path, outputs))
- all_inputs = list(map(escape_path, all_inputs))
+ out_outputs = [escape_path(x) for x in outputs]
+ all_inputs = [escape_path(x) for x in self._as_list(inputs)]
if implicit:
- implicit = map(escape_path, self._as_list(implicit))
+ implicit = [escape_path(x) for x in self._as_list(implicit)]
all_inputs.append('|')
all_inputs.extend(implicit)
if order_only:
- order_only = map(escape_path, self._as_list(order_only))
+ order_only = [escape_path(x) for x in self._as_list(order_only)]
all_inputs.append('||')
all_inputs.extend(order_only)
diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py
new file mode 100644
index 0000000..837007e
--- /dev/null
+++ b/misc/write_fake_manifests.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python
+
+"""Writes large manifest files, for manifest parser performance testing.
+
+The generated manifest files are (eerily) similar in appearance and size to the
+ones used in the Chromium project.
+
+Usage:
+ python misc/write_fake_manifests.py outdir # Will run for about 5s.
+
+The program contains a hardcoded random seed, so it will generate the same
+output every time it runs. By changing the seed, it's easy to generate many
+different sets of manifest files.
+"""
+
+import argparse
+import contextlib
+import os
+import random
+import sys
+
+import ninja_syntax
+
+
+def paretoint(avg, alpha):
+ """Returns a random integer that's avg on average, following a power law.
+ alpha determines the shape of the power curve. alpha has to be larger
+ than 1. The closer alpha is to 1, the higher the variation of the returned
+ numbers."""
+ return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1)))
+
+
+# Based on http://neugierig.org/software/chromium/class-name-generator.html
+def moar(avg_options, p_suffix):
+ kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url',
+ 'file', 'sync', 'content', 'http', 'profile']
+ kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref',
+ 'delegate', 'widget', 'proxy', 'stub', 'context',
+ 'manager', 'master', 'watcher', 'service', 'file', 'data',
+ 'resource', 'device', 'info', 'provider', 'internals', 'tracker',
+ 'api', 'layer']
+ kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest']
+ num_options = min(paretoint(avg_options, alpha=4), 5)
+ # The original allows kOption to repeat as long as no consecutive options
+ # repeat. This version doesn't allow any option repetition.
+ name = [random.choice(kStart)] + random.sample(kOption, num_options)
+ if random.random() < p_suffix:
+ name.append(random.choice(kOS))
+ return '_'.join(name)
+
+
+class GenRandom(object):
+ def __init__(self):
+ self.seen_names = set([None])
+ self.seen_defines = set([None])
+
+ def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1):
+ s = None
+ while s in seen:
+ s = moar(avg_options, p_suffix)
+ seen.add(s)
+ return s
+
+ def _n_unique_strings(self, n):
+ seen = set([None])
+ return [self._unique_string(seen, avg_options=3, p_suffix=0.4)
+ for _ in xrange(n)]
+
+ def target_name(self):
+ return self._unique_string(p_suffix=0, seen=self.seen_names)
+
+ def path(self):
+ return os.path.sep.join([
+ self._unique_string(self.seen_names, avg_options=1, p_suffix=0)
+ for _ in xrange(1 + paretoint(0.6, alpha=4))])
+
+ def src_obj_pairs(self, path, name):
+ num_sources = paretoint(55, alpha=2) + 1
+ return [(os.path.join('..', '..', path, s + '.cc'),
+ os.path.join('obj', path, '%s.%s.o' % (name, s)))
+ for s in self._n_unique_strings(num_sources)]
+
+ def defines(self):
+ return [
+ '-DENABLE_' + self._unique_string(self.seen_defines).upper()
+ for _ in xrange(paretoint(20, alpha=3))]
+
+
+LIB, EXE = 0, 1
+class Target(object):
+ def __init__(self, gen, kind):
+ self.name = gen.target_name()
+ self.dir_path = gen.path()
+ self.ninja_file_path = os.path.join(
+ 'obj', self.dir_path, self.name + '.ninja')
+ self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name)
+ if kind == LIB:
+ self.output = os.path.join('lib' + self.name + '.a')
+ elif kind == EXE:
+ self.output = os.path.join(self.name)
+ self.defines = gen.defines()
+ self.deps = []
+ self.kind = kind
+ self.has_compile_depends = random.random() < 0.4
+
+ @property
+ def includes(self):
+ return ['-I' + dep.dir_path for dep in self.deps]
+
+
+def write_target_ninja(ninja, target):
+ compile_depends = None
+ if target.has_compile_depends:
+ compile_depends = os.path.join(
+ 'obj', target.dir_path, target.name + '.stamp')
+ ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0])
+ ninja.newline()
+
+ ninja.variable('defines', target.defines)
+ if target.deps:
+ ninja.variable('includes', target.includes)
+ ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions'])
+ ninja.newline()
+
+ for src, obj in target.src_obj_pairs:
+ ninja.build(obj, 'cxx', src, implicit=compile_depends)
+ ninja.newline()
+
+ deps = [dep.output for dep in target.deps]
+ libs = [dep.output for dep in target.deps if dep.kind == LIB]
+ if target.kind == EXE:
+ ninja.variable('ldflags', '-Wl,pie')
+ ninja.variable('libs', libs)
+ link = { LIB: 'alink', EXE: 'link'}[target.kind]
+ ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs],
+ implicit=deps)
+
+
+def write_master_ninja(master_ninja, targets):
+ """Writes master build.ninja file, referencing all given subninjas."""
+ master_ninja.variable('cxx', 'c++')
+ master_ninja.variable('ld', '$cxx')
+ master_ninja.newline()
+
+ master_ninja.pool('link_pool', depth=4)
+ master_ninja.newline()
+
+ master_ninja.rule('cxx', description='CXX $out',
+ command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out',
+ depfile='$out.d', deps='gcc')
+ master_ninja.rule('alink', description='LIBTOOL-STATIC $out',
+ command='rm -f $out && libtool -static -o $out $in')
+ master_ninja.rule('link', description='LINK $out', pool='link_pool',
+ command='$ld $ldflags -o $out $in $libs')
+ master_ninja.rule('stamp', description='STAMP $out', command='touch $out')
+ master_ninja.newline()
+
+ for target in targets:
+ master_ninja.subninja(target.ninja_file_path)
+ master_ninja.newline()
+
+ master_ninja.comment('Short names for targets.')
+ for target in targets:
+ if target.name != target.output:
+ master_ninja.build(target.name, 'phony', target.output)
+ master_ninja.newline()
+
+ master_ninja.build('all', 'phony', [target.output for target in targets])
+ master_ninja.default('all')
+
+
+@contextlib.contextmanager
+def FileWriter(path):
+ """Context manager for a ninja_syntax object writing to a file."""
+ try:
+ os.makedirs(os.path.dirname(path))
+ except OSError:
+ pass
+ f = open(path, 'w')
+ yield ninja_syntax.Writer(f)
+ f.close()
+
+
+def random_targets():
+ num_targets = 800
+ gen = GenRandom()
+
+ # N-1 static libraries, and 1 executable depending on all of them.
+ targets = [Target(gen, LIB) for i in xrange(num_targets - 1)]
+ for i in range(len(targets)):
+ targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05]
+
+ last_target = Target(gen, EXE)
+ last_target.deps = targets[:]
+ last_target.src_obj_pairs = last_target.src_obj_pairs[0:10] # Trim.
+ targets.append(last_target)
+ return targets
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('outdir', help='output directory')
+ args = parser.parse_args()
+ root_dir = args.outdir
+
+ random.seed(12345)
+
+ targets = random_targets()
+ for target in targets:
+ with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n:
+ write_target_ninja(n, target)
+
+ with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja:
+ master_ninja.width = 120
+ write_master_ninja(master_ninja, targets)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/src/parser_perftest.cc b/src/depfile_parser_perftest.cc
index b215221..b215221 100644
--- a/src/parser_perftest.cc
+++ b/src/depfile_parser_perftest.cc
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 3233144..4dfae1a 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -14,6 +14,8 @@
#include "disk_interface.h"
+#include <algorithm>
+
#include <errno.h>
#include <stdio.h>
#include <string.h>
@@ -31,15 +33,16 @@ namespace {
string DirName(const string& path) {
#ifdef _WIN32
- const char kPathSeparator = '\\';
+ const char kPathSeparators[] = "\\/";
#else
- const char kPathSeparator = '/';
+ const char kPathSeparators[] = "/";
#endif
-
- string::size_type slash_pos = path.rfind(kPathSeparator);
+ string::size_type slash_pos = path.find_last_of(kPathSeparators);
if (slash_pos == string::npos)
return string(); // Nothing to do.
- while (slash_pos > 0 && path[slash_pos - 1] == kPathSeparator)
+ const char* const kEnd = kPathSeparators + strlen(kPathSeparators);
+ while (slash_pos > 0 &&
+ std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd)
--slash_pos;
return path.substr(0, slash_pos);
}
@@ -146,6 +149,9 @@ bool RealDiskInterface::WriteFile(const string& path, const string& contents) {
bool RealDiskInterface::MakeDir(const string& path) {
if (::MakeDir(path) < 0) {
+ if (errno == EEXIST) {
+ return true;
+ }
Error("mkdir(%s): %s", path.c_str(), strerror(errno));
return false;
}
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index 55822a6..51a1d14 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -93,7 +93,18 @@ TEST_F(DiskInterfaceTest, ReadFile) {
}
TEST_F(DiskInterfaceTest, MakeDirs) {
- EXPECT_TRUE(disk_.MakeDirs("path/with/double//slash/"));
+ string path = "path/with/double//slash/";
+ EXPECT_TRUE(disk_.MakeDirs(path.c_str()));
+ FILE* f = fopen((path + "a_file").c_str(), "w");
+ EXPECT_TRUE(f);
+ EXPECT_EQ(0, fclose(f));
+#ifdef _WIN32
+ string path2 = "another\\with\\back\\\\slashes\\";
+ EXPECT_TRUE(disk_.MakeDirs(path2.c_str()));
+ FILE* f2 = fopen((path2 + "a_file").c_str(), "w");
+ EXPECT_TRUE(f2);
+ EXPECT_EQ(0, fclose(f2));
+#endif
}
TEST_F(DiskInterfaceTest, RemoveFile) {
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index 17584dd..a566eda 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -296,16 +296,17 @@ bool ManifestParser::ParseEdge(string* err) {
if (!ExpectToken(Lexer::NEWLINE, err))
return false;
- // XXX scoped_ptr to handle error case.
- BindingEnv* env = new BindingEnv(env_);
-
- while (lexer_.PeekToken(Lexer::INDENT)) {
+ // Bindings on edges are rare, so allocate per-edge envs only when needed.
+ bool hasIdent = lexer_.PeekToken(Lexer::INDENT);
+ BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_;
+ while (hasIdent) {
string key;
EvalString val;
if (!ParseLet(&key, &val, err))
return false;
env->AddBinding(key, val.Evaluate(env_));
+ hasIdent = lexer_.PeekToken(Lexer::INDENT);
}
Edge* edge = state_->AddEdge(rule);
diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc
new file mode 100644
index 0000000..ed3a89a
--- /dev/null
+++ b/src/manifest_parser_perftest.cc
@@ -0,0 +1,112 @@
+// Copyright 2014 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Tests manifest parser performance. Expects to be run in ninja's root
+// directory.
+
+#include <numeric>
+#include <stdio.h>
+
+#ifdef _WIN32
+#include <direct.h>
+#else
+#include <unistd.h>
+#endif
+
+#include "disk_interface.h"
+#include "graph.h"
+#include "manifest_parser.h"
+#include "metrics.h"
+#include "state.h"
+#include "util.h"
+
+struct RealFileReader : public ManifestParser::FileReader {
+ virtual bool ReadFile(const string& path, string* content, string* err) {
+ return ::ReadFile(path, content, err) == 0;
+ }
+};
+
+bool WriteFakeManifests(const string& dir) {
+ RealDiskInterface disk_interface;
+ if (disk_interface.Stat(dir + "/build.ninja") > 0)
+ return true;
+
+ printf("Creating manifest data..."); fflush(stdout);
+ int err = system(("python misc/write_fake_manifests.py " + dir).c_str());
+ printf("done.\n");
+ return err == 0;
+}
+
+int LoadManifests(bool measure_command_evaluation) {
+ string err;
+ RealFileReader file_reader;
+ State state;
+ ManifestParser parser(&state, &file_reader);
+ if (!parser.Load("build.ninja", &err)) {
+ fprintf(stderr, "Failed to read test data: %s\n", err.c_str());
+ exit(1);
+ }
+ // Doing an empty build involves reading the manifest and evaluating all
+ // commands required for the requested targets. So include command
+ // evaluation in the perftest by default.
+ int optimization_guard = 0;
+ if (measure_command_evaluation)
+ for (size_t i = 0; i < state.edges_.size(); ++i)
+ optimization_guard += state.edges_[i]->EvaluateCommand().size();
+ return optimization_guard;
+}
+
+int main(int argc, char* argv[]) {
+ bool measure_command_evaluation = true;
+ int opt;
+ while ((opt = getopt(argc, argv, const_cast<char*>("fh"))) != -1) {
+ switch (opt) {
+ case 'f':
+ measure_command_evaluation = false;
+ break;
+ case 'h':
+ default:
+ printf("usage: manifest_parser_perftest\n"
+"\n"
+"options:\n"
+" -f only measure manifest load time, not command evaluation time\n"
+ );
+ return 1;
+ }
+ }
+
+ const char kManifestDir[] = "build/manifest_perftest";
+
+ if (!WriteFakeManifests(kManifestDir)) {
+ fprintf(stderr, "Failed to write test data\n");
+ return 1;
+ }
+
+ chdir(kManifestDir);
+
+ const int kNumRepetitions = 5;
+ vector<int> times;
+ for (int i = 0; i < kNumRepetitions; ++i) {
+ int64_t start = GetTimeMillis();
+ int optimization_guard = LoadManifests(measure_command_evaluation);
+ int delta = (int)(GetTimeMillis() - start);
+ printf("%dms (hash: %x)\n", delta, optimization_guard);
+ times.push_back(delta);
+ }
+
+ int min = *min_element(times.begin(), times.end());
+ int max = *max_element(times.begin(), times.end());
+ float total = accumulate(times.begin(), times.end(), 0);
+ printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size());
+}
diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc
index d2e2eb5..e465279 100644
--- a/src/msvc_helper-win32.cc
+++ b/src/msvc_helper-win32.cc
@@ -141,7 +141,7 @@ int CLWrapper::Run(const string& command, string* output) {
STARTUPINFO startup_info = {};
startup_info.cb = sizeof(STARTUPINFO);
startup_info.hStdInput = nul;
- startup_info.hStdError = stdout_write;
+ startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE);
startup_info.hStdOutput = stdout_write;
startup_info.dwFlags |= STARTF_USESTDHANDLES;
diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc
index 48fbe21..391c045 100644
--- a/src/msvc_helper_test.cc
+++ b/src/msvc_helper_test.cc
@@ -119,3 +119,10 @@ TEST(MSVCHelperTest, EnvBlock) {
cl.Run("cmd /c \"echo foo is %foo%", &output);
ASSERT_EQ("foo is bar\r\n", output);
}
+
+TEST(MSVCHelperTest, NoReadOfStderr) {
+ CLWrapper cl;
+ string output;
+ cl.Run("cmd /c \"echo to stdout&& echo to stderr 1>&2", &output);
+ ASSERT_EQ("to stdout\r\n", output);
+}
generated by cgit v0.12 at 2024-07-16 10:35:10 (GMT)