From 2832613dc7c1a4a8ff3b9df729954715762a8381 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Wed, 29 Jan 2014 21:57:58 -0800 Subject: Introduce the "console" pool This is a pre-defined pool with a depth of 1. It has the special property that any task in the pool has direct access to the console. This can be useful for interactive tasks or long-running tasks which produce status updates on the console (such as test suites). --- doc/manual.asciidoc | 17 +++++++++++++ src/build.cc | 14 +++++++++-- src/build_test.cc | 64 +++++++++++++++++++++++++++++++++++++++++-------- src/graph.cc | 4 ++++ src/graph.h | 1 + src/line_printer.cc | 51 ++++++++++++++++++++++++++++++++++----- src/line_printer.h | 20 ++++++++++++++++ src/manifest_parser.cc | 4 ++++ src/state.cc | 2 ++ src/state.h | 1 + src/subprocess-posix.cc | 50 +++++++++++++++++++++----------------- src/subprocess-win32.cc | 5 +++- src/subprocess.h | 8 +++++-- src/subprocess_test.cc | 15 ++++++++++++ 14 files changed, 213 insertions(+), 43 deletions(-) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 67fcbfd..5b0c1fe 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -648,6 +648,23 @@ build heavy_object2.obj: cc heavy_obj2.cc ---------------- +The `console` pool +^^^^^^^^^^^^^^^^^^ + +_Available since Ninja 1.5._ + +There exists a pre-defined pool named `console` with a depth of 1. It has +the special property that any task in the pool has direct access to the +standard input, output and error streams provided to Ninja, which are +normally connected to the user's console (hence the name) but could be +redirected. This can be useful for interactive tasks or long-running tasks +which produce status updates on the console (such as test suites). + +While a task in the `console` pool is running, Ninja's regular output (such +as progress status and output from concurrent tasks) is buffered until +it completes. + +This feature is not yet available on Windows. Ninja file reference -------------------- diff --git a/src/build.cc b/src/build.cc index f91ff2f..91f1754 100644 --- a/src/build.cc +++ b/src/build.cc @@ -97,6 +97,9 @@ void BuildStatus::BuildEdgeStarted(Edge* edge) { ++started_edges_; PrintStatus(edge); + + if (edge->use_console()) + printer_.SetConsoleLocked(true); } void BuildStatus::BuildEdgeFinished(Edge* edge, @@ -112,10 +115,13 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, *end_time = (int)(now - start_time_millis_); running_edges_.erase(i); + if (edge->use_console()) + printer_.SetConsoleLocked(false); + if (config_.verbosity == BuildConfig::QUIET) return; - if (printer_.is_smart_terminal()) + if (!edge->use_console() && printer_.is_smart_terminal()) PrintStatus(edge); // Print the command that is spewing before printing its output. @@ -145,6 +151,7 @@ void BuildStatus::BuildEdgeFinished(Edge* edge, } void BuildStatus::BuildFinished() { + printer_.SetConsoleLocked(false); printer_.PrintOnNewLine(""); } @@ -488,7 +495,7 @@ bool RealCommandRunner::CanRunMore() { bool RealCommandRunner::StartCommand(Edge* edge) { string command = edge->EvaluateCommand(); - Subprocess* subproc = subprocs_.Add(command); + Subprocess* subproc = subprocs_.Add(command, edge->use_console()); if (!subproc) return false; subproc_to_edge_.insert(make_pair(subproc, edge)); @@ -610,6 +617,7 @@ bool Builder::Build(string* err) { if (failures_allowed && command_runner_->CanRunMore()) { if (Edge* edge = plan_.FindWork()) { if (!StartEdge(edge, err)) { + Cleanup(); status_->BuildFinished(); return false; } @@ -630,6 +638,7 @@ bool Builder::Build(string* err) { CommandRunner::Result result; if (!command_runner_->WaitForCommand(&result) || result.status == ExitInterrupted) { + Cleanup(); status_->BuildFinished(); *err = "interrupted by user"; return false; @@ -637,6 +646,7 @@ bool Builder::Build(string* err) { --pending_commands; if (!FinishCommand(&result, err)) { + Cleanup(); status_->BuildFinished(); return false; } diff --git a/src/build_test.cc b/src/build_test.cc index 86a911b..119521e 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -44,6 +44,8 @@ struct PlanTest : public StateTestWithBuiltinRules { ASSERT_FALSE(plan_.FindWork()); sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp); } + + void TestPoolWithDepthOne(const char *test_case); }; TEST_F(PlanTest, Basic) { @@ -197,15 +199,8 @@ TEST_F(PlanTest, DependencyCycle) { ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } -TEST_F(PlanTest, PoolWithDepthOne) { - ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, -"pool foobar\n" -" depth = 1\n" -"rule poolcat\n" -" command = cat $in > $out\n" -" pool = foobar\n" -"build out1: poolcat in\n" -"build out2: poolcat in\n")); +void PlanTest::TestPoolWithDepthOne(const char* test_case) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, test_case)); GetNode("out1")->MarkDirty(); GetNode("out2")->MarkDirty(); string err; @@ -239,6 +234,28 @@ TEST_F(PlanTest, PoolWithDepthOne) { ASSERT_EQ(0, edge); } +TEST_F(PlanTest, PoolWithDepthOne) { + TestPoolWithDepthOne( +"pool foobar\n" +" depth = 1\n" +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = foobar\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} + +#ifndef _WIN32 +TEST_F(PlanTest, ConsolePool) { + TestPoolWithDepthOne( +"rule poolcat\n" +" command = cat $in > $out\n" +" pool = console\n" +"build out1: poolcat in\n" +"build out2: poolcat in\n"); +} +#endif + TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "pool foobar\n" @@ -515,7 +532,8 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { } } else if (edge->rule().name() == "true" || edge->rule().name() == "fail" || - edge->rule().name() == "interrupt") { + edge->rule().name() == "interrupt" || + edge->rule().name() == "console") { // Don't do anything. } else { printf("unknown command\n"); @@ -539,6 +557,15 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { return true; } + if (edge->rule().name() == "console") { + if (edge->use_console()) + result->status = ExitSuccess; + else + result->status = ExitFailure; + last_command_ = NULL; + return true; + } + if (edge->rule().name() == "fail") result->status = ExitFailure; else @@ -1911,3 +1938,20 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { RebuildTarget("out", manifest, "build_log", "ninja_deps2"); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); } + +TEST_F(BuildTest, Console) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule console\n" +" command = console\n" +" pool = console\n" +"build con: console in.txt\n")); + + fs_.Create("in.txt", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("con", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); +} diff --git a/src/graph.cc b/src/graph.cc index 65f9244..7121342 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -305,6 +305,10 @@ bool Edge::is_phony() const { return rule_ == &State::kPhonyRule; } +bool Edge::use_console() const { + return pool() == &State::kConsolePool; +} + void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %d%s, (:%s), ", prefix, path().c_str(), this, diff --git a/src/graph.h b/src/graph.h index 868413c..6cd7f25 100644 --- a/src/graph.h +++ b/src/graph.h @@ -183,6 +183,7 @@ struct Edge { } bool is_phony() const; + bool use_console() const; }; diff --git a/src/line_printer.cc b/src/line_printer.cc index 3537e88..ef1609c 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -26,7 +26,7 @@ #include "util.h" -LinePrinter::LinePrinter() : have_blank_line_(true) { +LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { #ifndef _WIN32 const char* term = getenv("TERM"); smart_terminal_ = isatty(1) && term && string(term) != "dumb"; @@ -43,6 +43,12 @@ LinePrinter::LinePrinter() : have_blank_line_(true) { } void LinePrinter::Print(string to_print, LineType type) { + if (console_locked_) { + line_buffer_ = to_print; + line_type_ = type; + return; + } + #ifdef _WIN32 CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); @@ -101,13 +107,46 @@ void LinePrinter::Print(string to_print, LineType type) { } } -void LinePrinter::PrintOnNewLine(const string& to_print) { - if (!have_blank_line_) - printf("\n"); - if (!to_print.empty()) { +void LinePrinter::PrintOrBuffer(const char* data, size_t size) { + if (console_locked_) { + output_buffer_.append(data, size); + } else { // Avoid printf and C strings, since the actual output might contain null // bytes like UTF-16 does (yuck). - fwrite(&to_print[0], sizeof(char), to_print.size(), stdout); + fwrite(data, 1, size, stdout); + } +} + +void LinePrinter::PrintOnNewLine(const string& to_print) { + if (console_locked_ && !line_buffer_.empty()) { + output_buffer_.append(line_buffer_); + output_buffer_.append(1, '\n'); + line_buffer_.clear(); + } + if (!have_blank_line_) { + PrintOrBuffer("\n", 1); + } + if (!to_print.empty()) { + PrintOrBuffer(&to_print[0], to_print.size()); } have_blank_line_ = to_print.empty() || *to_print.rbegin() == '\n'; } + +void LinePrinter::SetConsoleLocked(bool locked) { + if (locked == console_locked_) + return; + + if (locked) + PrintOnNewLine(""); + + console_locked_ = locked; + + if (!locked) { + PrintOnNewLine(output_buffer_); + if (!line_buffer_.empty()) { + Print(line_buffer_, line_type_); + } + output_buffer_.clear(); + line_buffer_.clear(); + } +} diff --git a/src/line_printer.h b/src/line_printer.h index aea2817..55225e5 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -15,6 +15,7 @@ #ifndef NINJA_LINE_PRINTER_H_ #define NINJA_LINE_PRINTER_H_ +#include #include using namespace std; @@ -37,6 +38,10 @@ struct LinePrinter { /// Prints a string on a new line, not overprinting previous output. void PrintOnNewLine(const string& to_print); + /// Lock or unlock the console. Any output sent to the LinePrinter while the + /// console is locked will not be printed until it is unlocked. + void SetConsoleLocked(bool locked); + private: /// Whether we can do fancy terminal control codes. bool smart_terminal_; @@ -44,9 +49,24 @@ struct LinePrinter { /// Whether the caret is at the beginning of a blank line. bool have_blank_line_; + /// Whether console is locked. + bool console_locked_; + + /// Buffered current line while console is locked. + string line_buffer_; + + /// Buffered line type while console is locked. + LineType line_type_; + + /// Buffered console output while console is locked. + string output_buffer_; + #ifdef _WIN32 void* console_; #endif + + /// Print the given data to the console, or buffer it if it is locked. + void PrintOrBuffer(const char *data, size_t size); }; #endif // NINJA_LINE_PRINTER_H_ diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 20be7f3..17584dd 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -316,6 +316,10 @@ bool ManifestParser::ParseEdge(string* err) { Pool* pool = state_->LookupPool(pool_name); if (pool == NULL) return lexer_.Error("unknown pool name '" + pool_name + "'", err); +#ifdef _WIN32 + if (pool == &State::kConsolePool) + return lexer_.Error("console pool unsupported on Windows", err); +#endif edge->pool_ = pool; } diff --git a/src/state.cc b/src/state.cc index 33f8423..7258272 100644 --- a/src/state.cc +++ b/src/state.cc @@ -69,11 +69,13 @@ bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { } Pool State::kDefaultPool("", 0); +Pool State::kConsolePool("console", 1); const Rule State::kPhonyRule("phony"); State::State() { AddRule(&kPhonyRule); AddPool(&kDefaultPool); + AddPool(&kConsolePool); } void State::AddRule(const Rule* rule) { diff --git a/src/state.h b/src/state.h index bcb0eff..c382dc0 100644 --- a/src/state.h +++ b/src/state.h @@ -82,6 +82,7 @@ struct Pool { /// Global state (file status, loaded rules) for a single run. struct State { static Pool kDefaultPool; + static Pool kConsolePool; static const Rule kPhonyRule; State(); diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index a9af756..793d48f 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -25,7 +25,8 @@ #include "util.h" -Subprocess::Subprocess() : fd_(-1), pid_(-1) { +Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1), + use_console_(use_console) { } Subprocess::~Subprocess() { if (fd_ >= 0) @@ -58,29 +59,31 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { // Track which fd we use to report errors on. int error_pipe = output_pipe[1]; do { - if (setpgid(0, 0) < 0) - break; - if (sigaction(SIGINT, &set->old_act_, 0) < 0) break; if (sigprocmask(SIG_SETMASK, &set->old_mask_, 0) < 0) break; - // Open /dev/null over stdin. - int devnull = open("/dev/null", O_RDONLY); - if (devnull < 0) - break; - if (dup2(devnull, 0) < 0) - break; - close(devnull); - - if (dup2(output_pipe[1], 1) < 0 || - dup2(output_pipe[1], 2) < 0) - break; - - // Now can use stderr for errors. - error_pipe = 2; - close(output_pipe[1]); + if (!use_console_) { + if (setpgid(0, 0) < 0) + break; + + // Open /dev/null over stdin. + int devnull = open("/dev/null", O_RDONLY); + if (devnull < 0) + break; + if (dup2(devnull, 0) < 0) + break; + close(devnull); + + if (dup2(output_pipe[1], 1) < 0 || + dup2(output_pipe[1], 2) < 0) + break; + + // Now can use stderr for errors. + error_pipe = 2; + close(output_pipe[1]); + } execl("/bin/sh", "/bin/sh", "-c", command.c_str(), (char *) NULL); } while (false); @@ -168,8 +171,8 @@ SubprocessSet::~SubprocessSet() { Fatal("sigprocmask: %s", strerror(errno)); } -Subprocess *SubprocessSet::Add(const string& command) { - Subprocess *subprocess = new Subprocess; +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + Subprocess *subprocess = new Subprocess(use_console); if (!subprocess->Start(this, command)) { delete subprocess; return 0; @@ -279,7 +282,10 @@ Subprocess* SubprocessSet::NextFinished() { void SubprocessSet::Clear() { for (vector::iterator i = running_.begin(); i != running_.end(); ++i) - kill(-(*i)->pid_, SIGINT); + // Since the foreground process is in our process group, it will receive a + // SIGINT at the same time as us. + if (!(*i)->use_console_) + kill(-(*i)->pid_, SIGINT); for (vector::iterator i = running_.begin(); i != running_.end(); ++i) delete *i; diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index 1b230b6..c9607e1 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -14,6 +14,7 @@ #include "subprocess.h" +#include #include #include @@ -213,7 +214,9 @@ BOOL WINAPI SubprocessSet::NotifyInterrupted(DWORD dwCtrlType) { return FALSE; } -Subprocess *SubprocessSet::Add(const string& command) { +Subprocess *SubprocessSet::Add(const string& command, bool use_console) { + assert(!use_console); // We don't support this yet on Windows. + Subprocess *subprocess = new Subprocess; if (!subprocess->Start(this, command)) { delete subprocess; diff --git a/src/subprocess.h b/src/subprocess.h index 4c1629c..6ea6f62 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -44,13 +44,14 @@ struct Subprocess { const string& GetOutput() const; private: - Subprocess(); bool Start(struct SubprocessSet* set, const string& command); void OnPipeReady(); string buf_; #ifdef _WIN32 + Subprocess(); + /// Set up pipe_ as the parent-side pipe of the subprocess; return the /// other end of the pipe, usable in the child process. HANDLE SetupPipe(HANDLE ioport); @@ -61,9 +62,12 @@ struct Subprocess { char overlapped_buf_[4 << 10]; bool is_reading_; #else + Subprocess(bool use_console); + int fd_; pid_t pid_; #endif + bool use_console_; friend struct SubprocessSet; }; @@ -75,7 +79,7 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess* Add(const string& command); + Subprocess* Add(const string& command, bool use_console = false); bool DoWork(); Subprocess* NextFinished(); void Clear(); diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 9f8dcea..775a13a 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -95,6 +95,21 @@ TEST_F(SubprocessTest, InterruptParent) { ADD_FAILURE() << "We should have been interrupted"; } +TEST_F(SubprocessTest, Console) { + // Skip test if we don't have the console ourselves. + if (isatty(0) && isatty(1) && isatty(2)) { + Subprocess* subproc = subprocs_.Add("test -t 0 -a -t 1 -a -t 2", + /*use_console=*/true); + ASSERT_NE((Subprocess *) 0, subproc); + + while (!subproc->Done()) { + subprocs_.DoWork(); + } + + EXPECT_EQ(ExitSuccess, subproc->Finish()); + } +} + #endif TEST_F(SubprocessTest, SetWithSingle) { -- cgit v0.12