From dcd41dcef3020e5c2cbe5c29b5b1d71e581de029 Mon Sep 17 00:00:00 2001 From: Evan Martin Date: Fri, 14 Nov 2014 10:53:33 -0800 Subject: add a --bootstrap mode for configure.py Instead of bootstrapping through a separate script, instead make configure.py able to either generate a build.ninja *or* just execute all the computed commands to build a ninja binary. --- configure.py | 128 ++++++++++++++++++++++++++++++++++++++++++++++++--- misc/ninja_syntax.py | 26 ++++++----- 2 files changed, 136 insertions(+), 18 deletions(-) diff --git a/configure.py b/configure.py index c2abcde..b818bfc 100755 --- a/configure.py +++ b/configure.py @@ -24,14 +24,90 @@ from __future__ import print_function from optparse import OptionParser import os import pipes +import string +import subprocess import sys import platform_helper sys.path.insert(0, 'misc') import ninja_syntax + +class Bootstrap: + """API shim for ninja_syntax.Writer that instead runs the commands. + + Used to bootstrap Ninja from scratch. In --bootstrap mode this + class is used to execute all the commands to build an executable. + It also proxies all calls to an underlying ninja_syntax.Writer, to + behave like non-bootstrap mode. + """ + def __init__(self, writer): + self.writer = writer + # Map of variable name => expanded variable value. + self.vars = {} + # Map of rule name => dict of rule attributes. + self.rules = { + 'phony': {} + } + + def comment(self, text): + return self.writer.comment(text) + + def newline(self): + return self.writer.newline() + + def variable(self, key, val): + self.vars[key] = self._expand(val) + return self.writer.variable(key, val) + + def rule(self, name, **kwargs): + self.rules[name] = kwargs + return self.writer.rule(name, **kwargs) + + def build(self, outputs, rule, inputs=None, **kwargs): + ruleattr = self.rules[rule] + cmd = ruleattr.get('command') + if cmd is None: # A phony rule, for example. + return + + # Implement just enough of Ninja variable expansion etc. to + # make the bootstrap build work. + local_vars = { + 'in': self._expand_paths(inputs), + 'out': self._expand_paths(outputs) + } + for key, val in kwargs.get('variables', []): + local_vars[key] = ' '.join(ninja_syntax.as_list(val)) + + self._run_command(self._expand(cmd, local_vars)) + + return self.writer.build(outputs, rule, inputs, **kwargs) + + def default(self, paths): + return self.writer.default(paths) + + def _expand_paths(self, paths): + """Expand $vars in an array of paths, e.g. from a 'build' block.""" + paths = ninja_syntax.as_list(paths) + return ' '.join(map(self._expand, paths)) + + def _expand(self, str, local_vars={}): + """Expand $vars in a string.""" + return ninja_syntax.expand(str, self.vars, local_vars) + + def _run_command(self, cmdline): + """Run a subcommand, quietly. Prints the full command on error.""" + try: + subprocess.check_call(cmdline, shell=True) + except subprocess.CalledProcessError, e: + print('when running: ', cmdline) + raise + + parser = OptionParser() profilers = ['gmon', 'pprof'] +parser.add_option('--bootstrap', action='store_true', + help='bootstrap a ninja binary from nothing') parser.add_option('--platform', help='target platform (' + '/'.join(platform_helper.platforms()) + ')', @@ -64,8 +140,20 @@ else: host = platform BUILD_FILENAME = 'build.ninja' -buildfile = open(BUILD_FILENAME, 'w') -n = ninja_syntax.Writer(buildfile) +ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w')) +n = ninja_writer + +if options.bootstrap: + # Make the build directory. + try: + os.mkdir('build') + except OSError: + pass + # Wrap ninja_writer with the Bootstrapper, which also executes the + # commands. + print('bootstrapping ninja...') + n = Bootstrap(n) + n.comment('This file is used to build ninja itself.') n.comment('It is generated by ' + os.path.basename(__file__) + '.') n.newline() @@ -74,7 +162,10 @@ n.variable('ninja_required_version', '1.3') n.newline() n.comment('The arguments passed to configure.py, for rerunning it.') -n.variable('configure_args', ' '.join(sys.argv[1:])) +configure_args = sys.argv[1:] +if '--bootstrap' in configure_args: + configure_args.remove('--bootstrap') +n.variable('configure_args', ' '.join(configure_args)) env_keys = set(['CXX', 'AR', 'CFLAGS', 'LDFLAGS']) configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys) if configure_env: @@ -114,7 +205,8 @@ else: n.variable('ar', configure_env.get('AR', 'ar')) if platform.is_msvc(): - cflags = ['/nologo', # Don't print startup banner. + cflags = ['/showIncludes', + '/nologo', # Don't print startup banner. '/Zi', # Create pdb with debug info. '/W4', # Highest warning level. '/WX', # Warnings as errors. @@ -129,6 +221,10 @@ if platform.is_msvc(): '/DNOMINMAX', '/D_CRT_SECURE_NO_WARNINGS', '/D_VARIADIC_MAX=10', '/DNINJA_PYTHON="%s"' % options.with_python] + if options.bootstrap: + # In bootstrap mode, we have no ninja process to catch /showIncludes + # output. + cflags.remove('/showIncludes') if platform.msvc_needs_fs(): cflags.append('/FS') ldflags = ['/DEBUG', '/libpath:$builddir'] @@ -200,9 +296,10 @@ n.newline() if platform.is_msvc(): n.rule('cxx', - command='$cxx /showIncludes $cflags -c $in /Fo$out', + command='$cxx $cflags -c $in /Fo$out', description='CXX $out', - deps='msvc') + deps='msvc' # /showIncludes is included in $cflags. + ) else: n.rule('cxx', command='$cxx -MMD -MT $out -MF $out.d $cflags -c $in -o $out', @@ -252,7 +349,6 @@ if have_browse: n.comment('the depfile parser and ninja lexers are generated using re2c.') def has_re2c(): - import subprocess try: proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE) return int(proc.communicate()[0], 10) >= 1103 @@ -321,6 +417,12 @@ ninja = n.build(binary('ninja'), 'link', objs, implicit=ninja_lib, n.newline() all_targets += ninja +if options.bootstrap: + # We've built the ninja binary. Don't run any more commands + # through the bootstrap executor, but continue writing the + # build.ninja file. + n = ninja_writer + n.comment('Tests all build into ninja_test executable.') test_libs = libs @@ -434,4 +536,16 @@ if host.is_linux(): n.build('all', 'phony', all_targets) +n.close() print('wrote %s.' % BUILD_FILENAME) + +if options.bootstrap: + print('bootstrap complete. rebuilding...') + if platform.is_windows(): + bootstrap_exe = 'ninja.bootstrap.exe' + if os.path.exists(bootstrap_exe): + os.unlink(bootstrap_exe) + os.rename('ninja.exe', bootstrap_exe) + subprocess.check_call('ninja.bootstrap.exe', shell=True) + else: + subprocess.check_call('./ninja', shell=True) diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index e200514..8673518 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -60,16 +60,16 @@ class Writer(object): def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, variables=None): - outputs = self._as_list(outputs) + outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] - all_inputs = [escape_path(x) for x in self._as_list(inputs)] + all_inputs = [escape_path(x) for x in as_list(inputs)] if implicit: - implicit = [escape_path(x) for x in self._as_list(implicit)] + implicit = [escape_path(x) for x in as_list(implicit)] all_inputs.append('|') all_inputs.extend(implicit) if order_only: - order_only = [escape_path(x) for x in self._as_list(order_only)] + order_only = [escape_path(x) for x in as_list(order_only)] all_inputs.append('||') all_inputs.extend(order_only) @@ -94,7 +94,7 @@ class Writer(object): self._line('subninja %s' % path) def default(self, paths): - self._line('default %s' % ' '.join(self._as_list(paths))) + self._line('default %s' % ' '.join(as_list(paths))) def _count_dollars_before_index(self, s, i): """Returns the number of '$' characters right in front of s[i].""" @@ -141,12 +141,16 @@ class Writer(object): self.output.write(leading_space + text + '\n') - def _as_list(self, input): - if input is None: - return [] - if isinstance(input, list): - return input - return [input] + def close(self): + self.output.close() + + +def as_list(input): + if input is None: + return [] + if isinstance(input, list): + return input + return [input] def escape(string): -- cgit v0.12