diff options
-rw-r--r-- | .clang-format | 25 | ||||
-rw-r--r-- | .gitignore | 3 | ||||
-rwxr-xr-x | configure.py | 9 | ||||
-rw-r--r-- | misc/bash-completion | 4 | ||||
-rw-r--r-- | misc/ninja_syntax.py | 9 | ||||
-rw-r--r-- | misc/write_fake_manifests.py | 219 | ||||
-rw-r--r-- | src/depfile_parser_perftest.cc (renamed from src/parser_perftest.cc) | 0 | ||||
-rw-r--r-- | src/disk_interface.cc | 16 | ||||
-rw-r--r-- | src/disk_interface_test.cc | 13 | ||||
-rw-r--r-- | src/manifest_parser.cc | 9 | ||||
-rw-r--r-- | src/manifest_parser_perftest.cc | 112 | ||||
-rw-r--r-- | src/msvc_helper-win32.cc | 2 | ||||
-rw-r--r-- | src/msvc_helper_test.cc | 7 |
13 files changed, 406 insertions, 22 deletions
diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..1841c03 --- /dev/null +++ b/.clang-format @@ -0,0 +1,25 @@ +# Copyright 2014 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This isn't meant to be authoritative, but it's good enough to be useful. +# Still use your best judgement for formatting decisions: clang-format +# sometimes makes strange choices. + +BasedOnStyle: Google +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +ConstructorInitializerAllOnOneLineOrOnePerLine: false +Cpp11BracedListStyle: false +IndentCaseLabels: false @@ -8,9 +8,10 @@ TAGS /ninja /build_log_perftest /canon_perftest +/depfile_parser_perftest /hash_collision_bench /ninja_test -/parser_perftest +/manifest_parser_perftest /graph.png /doc/manual.html /doc/doxygen diff --git a/configure.py b/configure.py index da2f6ef..c5a6abd 100755 --- a/configure.py +++ b/configure.py @@ -377,18 +377,21 @@ all_targets += ninja_test n.comment('Ancillary executables.') -objs = cxx('parser_perftest') -all_targets += n.build(binary('parser_perftest'), 'link', objs, - implicit=ninja_lib, variables=[('libs', libs)]) objs = cxx('build_log_perftest') all_targets += n.build(binary('build_log_perftest'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) objs = cxx('canon_perftest') all_targets += n.build(binary('canon_perftest'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) +objs = cxx('depfile_parser_perftest') +all_targets += n.build(binary('depfile_parser_perftest'), 'link', objs, + implicit=ninja_lib, variables=[('libs', libs)]) objs = cxx('hash_collision_bench') all_targets += n.build(binary('hash_collision_bench'), 'link', objs, implicit=ninja_lib, variables=[('libs', libs)]) +objs = cxx('manifest_parser_perftest') +all_targets += n.build(binary('manifest_parser_perftest'), 'link', objs, + implicit=ninja_lib, variables=[('libs', libs)]) n.newline() n.comment('Generate a graph using the "graph" tool.') diff --git a/misc/bash-completion b/misc/bash-completion index 2d6975b..719e7a8 100644 --- a/misc/bash-completion +++ b/misc/bash-completion @@ -26,12 +26,12 @@ _ninja_target() { dir="." line=$(echo ${COMP_LINE} | cut -d" " -f 2-) # filter out all non relevant arguments but keep C for dirs - while getopts C:f:j:l:k:nvd:t: opt "${line[@]}"; do + while getopts C:f:j:l:k:nvd:t: opt $line; do case $opt in C) dir="$OPTARG" ;; esac done; - targets_command="ninja -C ${dir} -t targets all" + targets_command="eval ninja -C \"${dir}\" -t targets all" targets=$((${targets_command} 2>/dev/null) | awk -F: '{print $1}') COMPREPLY=($(compgen -W "$targets" -- "$cur")) fi diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index d69e3e4..4b9b547 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -61,16 +61,15 @@ class Writer(object): def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, variables=None): outputs = self._as_list(outputs) - all_inputs = self._as_list(inputs)[:] - out_outputs = list(map(escape_path, outputs)) - all_inputs = list(map(escape_path, all_inputs)) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in self._as_list(inputs)] if implicit: - implicit = map(escape_path, self._as_list(implicit)) + implicit = [escape_path(x) for x in self._as_list(implicit)] all_inputs.append('|') all_inputs.extend(implicit) if order_only: - order_only = map(escape_path, self._as_list(order_only)) + order_only = [escape_path(x) for x in self._as_list(order_only)] all_inputs.append('||') all_inputs.extend(order_only) diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py new file mode 100644 index 0000000..837007e --- /dev/null +++ b/misc/write_fake_manifests.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python + +"""Writes large manifest files, for manifest parser performance testing. + +The generated manifest files are (eerily) similar in appearance and size to the +ones used in the Chromium project. + +Usage: + python misc/write_fake_manifests.py outdir # Will run for about 5s. + +The program contains a hardcoded random seed, so it will generate the same +output every time it runs. By changing the seed, it's easy to generate many +different sets of manifest files. +""" + +import argparse +import contextlib +import os +import random +import sys + +import ninja_syntax + + +def paretoint(avg, alpha): + """Returns a random integer that's avg on average, following a power law. + alpha determines the shape of the power curve. alpha has to be larger + than 1. The closer alpha is to 1, the higher the variation of the returned + numbers.""" + return int(random.paretovariate(alpha) * avg / (alpha / (alpha - 1))) + + +# Based on http://neugierig.org/software/chromium/class-name-generator.html +def moar(avg_options, p_suffix): + kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url', + 'file', 'sync', 'content', 'http', 'profile'] + kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref', + 'delegate', 'widget', 'proxy', 'stub', 'context', + 'manager', 'master', 'watcher', 'service', 'file', 'data', + 'resource', 'device', 'info', 'provider', 'internals', 'tracker', + 'api', 'layer'] + kOS = ['win', 'mac', 'aura', 'linux', 'android', 'unittest', 'browsertest'] + num_options = min(paretoint(avg_options, alpha=4), 5) + # The original allows kOption to repeat as long as no consecutive options + # repeat. This version doesn't allow any option repetition. + name = [random.choice(kStart)] + random.sample(kOption, num_options) + if random.random() < p_suffix: + name.append(random.choice(kOS)) + return '_'.join(name) + + +class GenRandom(object): + def __init__(self): + self.seen_names = set([None]) + self.seen_defines = set([None]) + + def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1): + s = None + while s in seen: + s = moar(avg_options, p_suffix) + seen.add(s) + return s + + def _n_unique_strings(self, n): + seen = set([None]) + return [self._unique_string(seen, avg_options=3, p_suffix=0.4) + for _ in xrange(n)] + + def target_name(self): + return self._unique_string(p_suffix=0, seen=self.seen_names) + + def path(self): + return os.path.sep.join([ + self._unique_string(self.seen_names, avg_options=1, p_suffix=0) + for _ in xrange(1 + paretoint(0.6, alpha=4))]) + + def src_obj_pairs(self, path, name): + num_sources = paretoint(55, alpha=2) + 1 + return [(os.path.join('..', '..', path, s + '.cc'), + os.path.join('obj', path, '%s.%s.o' % (name, s))) + for s in self._n_unique_strings(num_sources)] + + def defines(self): + return [ + '-DENABLE_' + self._unique_string(self.seen_defines).upper() + for _ in xrange(paretoint(20, alpha=3))] + + +LIB, EXE = 0, 1 +class Target(object): + def __init__(self, gen, kind): + self.name = gen.target_name() + self.dir_path = gen.path() + self.ninja_file_path = os.path.join( + 'obj', self.dir_path, self.name + '.ninja') + self.src_obj_pairs = gen.src_obj_pairs(self.dir_path, self.name) + if kind == LIB: + self.output = os.path.join('lib' + self.name + '.a') + elif kind == EXE: + self.output = os.path.join(self.name) + self.defines = gen.defines() + self.deps = [] + self.kind = kind + self.has_compile_depends = random.random() < 0.4 + + @property + def includes(self): + return ['-I' + dep.dir_path for dep in self.deps] + + +def write_target_ninja(ninja, target): + compile_depends = None + if target.has_compile_depends: + compile_depends = os.path.join( + 'obj', target.dir_path, target.name + '.stamp') + ninja.build(compile_depends, 'stamp', target.src_obj_pairs[0][0]) + ninja.newline() + + ninja.variable('defines', target.defines) + if target.deps: + ninja.variable('includes', target.includes) + ninja.variable('cflags', ['-Wall', '-fno-rtti', '-fno-exceptions']) + ninja.newline() + + for src, obj in target.src_obj_pairs: + ninja.build(obj, 'cxx', src, implicit=compile_depends) + ninja.newline() + + deps = [dep.output for dep in target.deps] + libs = [dep.output for dep in target.deps if dep.kind == LIB] + if target.kind == EXE: + ninja.variable('ldflags', '-Wl,pie') + ninja.variable('libs', libs) + link = { LIB: 'alink', EXE: 'link'}[target.kind] + ninja.build(target.output, link, [obj for _, obj in target.src_obj_pairs], + implicit=deps) + + +def write_master_ninja(master_ninja, targets): + """Writes master build.ninja file, referencing all given subninjas.""" + master_ninja.variable('cxx', 'c++') + master_ninja.variable('ld', '$cxx') + master_ninja.newline() + + master_ninja.pool('link_pool', depth=4) + master_ninja.newline() + + master_ninja.rule('cxx', description='CXX $out', + command='$cxx -MMD -MF $out.d $defines $includes $cflags -c $in -o $out', + depfile='$out.d', deps='gcc') + master_ninja.rule('alink', description='LIBTOOL-STATIC $out', + command='rm -f $out && libtool -static -o $out $in') + master_ninja.rule('link', description='LINK $out', pool='link_pool', + command='$ld $ldflags -o $out $in $libs') + master_ninja.rule('stamp', description='STAMP $out', command='touch $out') + master_ninja.newline() + + for target in targets: + master_ninja.subninja(target.ninja_file_path) + master_ninja.newline() + + master_ninja.comment('Short names for targets.') + for target in targets: + if target.name != target.output: + master_ninja.build(target.name, 'phony', target.output) + master_ninja.newline() + + master_ninja.build('all', 'phony', [target.output for target in targets]) + master_ninja.default('all') + + +@contextlib.contextmanager +def FileWriter(path): + """Context manager for a ninja_syntax object writing to a file.""" + try: + os.makedirs(os.path.dirname(path)) + except OSError: + pass + f = open(path, 'w') + yield ninja_syntax.Writer(f) + f.close() + + +def random_targets(): + num_targets = 800 + gen = GenRandom() + + # N-1 static libraries, and 1 executable depending on all of them. + targets = [Target(gen, LIB) for i in xrange(num_targets - 1)] + for i in range(len(targets)): + targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05] + + last_target = Target(gen, EXE) + last_target.deps = targets[:] + last_target.src_obj_pairs = last_target.src_obj_pairs[0:10] # Trim. + targets.append(last_target) + return targets + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('outdir', help='output directory') + args = parser.parse_args() + root_dir = args.outdir + + random.seed(12345) + + targets = random_targets() + for target in targets: + with FileWriter(os.path.join(root_dir, target.ninja_file_path)) as n: + write_target_ninja(n, target) + + with FileWriter(os.path.join(root_dir, 'build.ninja')) as master_ninja: + master_ninja.width = 120 + write_master_ninja(master_ninja, targets) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/parser_perftest.cc b/src/depfile_parser_perftest.cc index b215221..b215221 100644 --- a/src/parser_perftest.cc +++ b/src/depfile_parser_perftest.cc diff --git a/src/disk_interface.cc b/src/disk_interface.cc index 3233144..4dfae1a 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -14,6 +14,8 @@ #include "disk_interface.h" +#include <algorithm> + #include <errno.h> #include <stdio.h> #include <string.h> @@ -31,15 +33,16 @@ namespace { string DirName(const string& path) { #ifdef _WIN32 - const char kPathSeparator = '\\'; + const char kPathSeparators[] = "\\/"; #else - const char kPathSeparator = '/'; + const char kPathSeparators[] = "/"; #endif - - string::size_type slash_pos = path.rfind(kPathSeparator); + string::size_type slash_pos = path.find_last_of(kPathSeparators); if (slash_pos == string::npos) return string(); // Nothing to do. - while (slash_pos > 0 && path[slash_pos - 1] == kPathSeparator) + const char* const kEnd = kPathSeparators + strlen(kPathSeparators); + while (slash_pos > 0 && + std::find(kPathSeparators, kEnd, path[slash_pos - 1]) != kEnd) --slash_pos; return path.substr(0, slash_pos); } @@ -146,6 +149,9 @@ bool RealDiskInterface::WriteFile(const string& path, const string& contents) { bool RealDiskInterface::MakeDir(const string& path) { if (::MakeDir(path) < 0) { + if (errno == EEXIST) { + return true; + } Error("mkdir(%s): %s", path.c_str(), strerror(errno)); return false; } diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index 55822a6..51a1d14 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -93,7 +93,18 @@ TEST_F(DiskInterfaceTest, ReadFile) { } TEST_F(DiskInterfaceTest, MakeDirs) { - EXPECT_TRUE(disk_.MakeDirs("path/with/double//slash/")); + string path = "path/with/double//slash/"; + EXPECT_TRUE(disk_.MakeDirs(path.c_str())); + FILE* f = fopen((path + "a_file").c_str(), "w"); + EXPECT_TRUE(f); + EXPECT_EQ(0, fclose(f)); +#ifdef _WIN32 + string path2 = "another\\with\\back\\\\slashes\\"; + EXPECT_TRUE(disk_.MakeDirs(path2.c_str())); + FILE* f2 = fopen((path2 + "a_file").c_str(), "w"); + EXPECT_TRUE(f2); + EXPECT_EQ(0, fclose(f2)); +#endif } TEST_F(DiskInterfaceTest, RemoveFile) { diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 17584dd..a566eda 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -296,16 +296,17 @@ bool ManifestParser::ParseEdge(string* err) { if (!ExpectToken(Lexer::NEWLINE, err)) return false; - // XXX scoped_ptr to handle error case. - BindingEnv* env = new BindingEnv(env_); - - while (lexer_.PeekToken(Lexer::INDENT)) { + // Bindings on edges are rare, so allocate per-edge envs only when needed. + bool hasIdent = lexer_.PeekToken(Lexer::INDENT); + BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_; + while (hasIdent) { string key; EvalString val; if (!ParseLet(&key, &val, err)) return false; env->AddBinding(key, val.Evaluate(env_)); + hasIdent = lexer_.PeekToken(Lexer::INDENT); } Edge* edge = state_->AddEdge(rule); diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc new file mode 100644 index 0000000..ed3a89a --- /dev/null +++ b/src/manifest_parser_perftest.cc @@ -0,0 +1,112 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests manifest parser performance. Expects to be run in ninja's root +// directory. + +#include <numeric> +#include <stdio.h> + +#ifdef _WIN32 +#include <direct.h> +#else +#include <unistd.h> +#endif + +#include "disk_interface.h" +#include "graph.h" +#include "manifest_parser.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +struct RealFileReader : public ManifestParser::FileReader { + virtual bool ReadFile(const string& path, string* content, string* err) { + return ::ReadFile(path, content, err) == 0; + } +}; + +bool WriteFakeManifests(const string& dir) { + RealDiskInterface disk_interface; + if (disk_interface.Stat(dir + "/build.ninja") > 0) + return true; + + printf("Creating manifest data..."); fflush(stdout); + int err = system(("python misc/write_fake_manifests.py " + dir).c_str()); + printf("done.\n"); + return err == 0; +} + +int LoadManifests(bool measure_command_evaluation) { + string err; + RealFileReader file_reader; + State state; + ManifestParser parser(&state, &file_reader); + if (!parser.Load("build.ninja", &err)) { + fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); + exit(1); + } + // Doing an empty build involves reading the manifest and evaluating all + // commands required for the requested targets. So include command + // evaluation in the perftest by default. + int optimization_guard = 0; + if (measure_command_evaluation) + for (size_t i = 0; i < state.edges_.size(); ++i) + optimization_guard += state.edges_[i]->EvaluateCommand().size(); + return optimization_guard; +} + +int main(int argc, char* argv[]) { + bool measure_command_evaluation = true; + int opt; + while ((opt = getopt(argc, argv, const_cast<char*>("fh"))) != -1) { + switch (opt) { + case 'f': + measure_command_evaluation = false; + break; + case 'h': + default: + printf("usage: manifest_parser_perftest\n" +"\n" +"options:\n" +" -f only measure manifest load time, not command evaluation time\n" + ); + return 1; + } + } + + const char kManifestDir[] = "build/manifest_perftest"; + + if (!WriteFakeManifests(kManifestDir)) { + fprintf(stderr, "Failed to write test data\n"); + return 1; + } + + chdir(kManifestDir); + + const int kNumRepetitions = 5; + vector<int> times; + for (int i = 0; i < kNumRepetitions; ++i) { + int64_t start = GetTimeMillis(); + int optimization_guard = LoadManifests(measure_command_evaluation); + int delta = (int)(GetTimeMillis() - start); + printf("%dms (hash: %x)\n", delta, optimization_guard); + times.push_back(delta); + } + + int min = *min_element(times.begin(), times.end()); + int max = *max_element(times.begin(), times.end()); + float total = accumulate(times.begin(), times.end(), 0); + printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); +} diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index d2e2eb5..e465279 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -141,7 +141,7 @@ int CLWrapper::Run(const string& command, string* output) { STARTUPINFO startup_info = {}; startup_info.cb = sizeof(STARTUPINFO); startup_info.hStdInput = nul; - startup_info.hStdError = stdout_write; + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); startup_info.hStdOutput = stdout_write; startup_info.dwFlags |= STARTF_USESTDHANDLES; diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index 48fbe21..391c045 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -119,3 +119,10 @@ TEST(MSVCHelperTest, EnvBlock) { cl.Run("cmd /c \"echo foo is %foo%", &output); ASSERT_EQ("foo is bar\r\n", output); } + +TEST(MSVCHelperTest, NoReadOfStderr) { + CLWrapper cl; + string output; + cl.Run("cmd /c \"echo to stdout&& echo to stderr 1>&2", &output); + ASSERT_EQ("to stdout\r\n", output); +} |