From 8f87ee0469820d247fa20971c07dbad5f2aa744e Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Sat, 16 Nov 2019 16:09:06 +0100 Subject: Add restat tool which recalculates all mtimes in the build log --- src/build_log.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ src/build_log.h | 5 +++++ src/build_log_test.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ src/ninja.cc | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) diff --git a/src/build_log.cc b/src/build_log.cc index c4a08a0..0b06cc5 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -21,6 +21,7 @@ #endif #include "build_log.h" +#include "disk_interface.h" #include #include @@ -418,3 +419,50 @@ bool BuildLog::Recompact(const string& path, const BuildLogUser& user, return true; } + +bool BuildLog::Restat(const StringPiece path, + const DiskInterface& disk_interface, + std::string* const err) { + METRIC_RECORD(".ninja_log restat"); + + Close(); + std::string temp_path = path.AsString() + ".restat"; + FILE* f = fopen(temp_path.c_str(), "wb"); + if (!f) { + *err = strerror(errno); + return false; + } + + if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { + *err = strerror(errno); + fclose(f); + return false; + } + for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { + const TimeStamp mtime = disk_interface.Stat(i->second->output, err); + if (mtime == -1) { + fclose(f); + return false; + } + i->second->mtime = mtime; + + if (!WriteEntry(f, *i->second)) { + *err = strerror(errno); + fclose(f); + return false; + } + } + + fclose(f); + if (unlink(path.str_) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.str_) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} diff --git a/src/build_log.h b/src/build_log.h index 5268fab..d52dd3b 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -23,6 +23,7 @@ using namespace std; #include "timestamp.h" #include "util.h" // uint64_t +struct DiskInterface; struct Edge; /// Can answer questions about the manifest for the BuildLog. @@ -81,6 +82,10 @@ struct BuildLog { /// Rewrite the known log entries, throwing away old data. bool Recompact(const string& path, const BuildLogUser& user, string* err); + /// Restat all outputs in the log + bool Restat(StringPiece path, const DiskInterface& disk_interface, + std::string* err); + typedef ExternalStringHashMap::Type Entries; const Entries& entries() const { return entries_; } diff --git a/src/build_log_test.cc b/src/build_log_test.cc index ad30380..eee8290 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -25,6 +25,7 @@ #include #include #endif +#include namespace { @@ -216,6 +217,47 @@ TEST_F(BuildLogTest, DuplicateVersionHeader) { ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); } +struct TestDiskInterface : public DiskInterface { + virtual TimeStamp Stat(const string& path, string* err) const { + return 4; + } + virtual bool WriteFile(const string& path, const string& contents) { + assert(false); + return true; + } + virtual bool MakeDir(const string& path) { + assert(false); + return false; + } + virtual Status ReadFile(const string& path, string* contents, string* err) { + assert(false); + return NotFound; + } + virtual int RemoveFile(const string& path) { + assert(false); + return 0; + } +}; + +TEST_F(BuildLogTest, Restat) { + FILE* f = fopen(kTestFilename, "wb"); + fprintf(f, "# ninja log v4\n" + "1\t2\t3\tout\tcommand\n"); + fclose(f); + std::string err; + BuildLog log; + EXPECT_TRUE(log.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + BuildLog::LogEntry* e = log.LookupByOutput("out"); + ASSERT_EQ(3, e->mtime); + + TestDiskInterface testDiskInterface; + EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, &err)); + ASSERT_EQ("", err); + e = log.LookupByOutput("out"); + ASSERT_EQ(4, e->mtime); +} + TEST_F(BuildLogTest, VeryLongInputLine) { // Ninja's build log buffer is currently 256kB. Lines longer than that are // silently ignored, but don't affect parsing of other lines. diff --git a/src/ninja.cc b/src/ninja.cc index c24f09d..8b76ded 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -125,6 +125,7 @@ struct NinjaMain : public BuildLogUser { int ToolClean(const Options* options, int argc, char* argv[]); int ToolCompilationDatabase(const Options* options, int argc, char* argv[]); int ToolRecompact(const Options* options, int argc, char* argv[]); + int ToolRestat(const Options* options, int argc, char* argv[]); int ToolUrtle(const Options* options, int argc, char** argv); int ToolRules(const Options* options, int argc, char* argv[]); @@ -852,6 +853,41 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) { return 0; } +int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) { + if (!EnsureBuildDirExists()) + return 1; + + string log_path = ".ninja_log"; + if (!build_dir_.empty()) + log_path = build_dir_ + "/" + log_path; + + string err; + if (!build_log_.Load(log_path, &err)) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + return EXIT_FAILURE; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning true. + Warning("%s", err.c_str()); + err.clear(); + } + + bool success = build_log_.Restat(log_path, disk_interface_, &err); + if (!success) { + Error("failed recompaction: %s", err.c_str()); + return EXIT_FAILURE; + } + + if (!config_.dry_run) { + if (!build_log_.OpenForWrite(log_path, *this, &err)) { + Error("opening build log: %s", err.c_str()); + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} + int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) { // RLE encoded. const char* urtle = @@ -904,6 +940,8 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase }, { "recompact", "recompacts ninja-internal data structures", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact }, + { "restat", "restats all outputs in the build log", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRestat }, { "rules", "list all rules", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules }, { "urtle", NULL, -- cgit v0.12 From 791c887e22046e5e7a2d05ecb5ff27701d56895d Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Fri, 20 Dec 2019 15:49:57 +0100 Subject: Ignore nonexistent .ninja_log/.ninja_deps for restat and recompact --- misc/output_test.py | 5 +++++ src/build_log.cc | 12 ++++++------ src/build_log.h | 3 ++- src/build_log_test.cc | 2 +- src/deps_log.cc | 14 +++++++------- src/deps_log.h | 3 ++- src/load_status.h | 24 ++++++++++++++++++++++++ src/ninja.cc | 34 ++++++++++++++++++++++++---------- 8 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 src/load_status.h diff --git a/misc/output_test.py b/misc/output_test.py index 966417d..3fd9c32 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -102,5 +102,10 @@ red \x1b[31mred\x1b[0m ''') + def test_pr_1685(self): + # Running those tools without .ninja_deps and .ninja_log shouldn't fail. + self.assertEqual(run('', flags='-t recompact'), '') + self.assertEqual(run('', flags='-t restat'), '') + if __name__ == '__main__': unittest.main() diff --git a/src/build_log.cc b/src/build_log.cc index 0b06cc5..e2a9344 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -242,14 +242,14 @@ struct LineReader { char* line_end_; }; -bool BuildLog::Load(const string& path, string* err) { +LoadStatus BuildLog::Load(const string& path, string* err) { METRIC_RECORD(".ninja_log load"); FILE* file = fopen(path.c_str(), "r"); if (!file) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } int log_version = 0; @@ -270,7 +270,7 @@ bool BuildLog::Load(const string& path, string* err) { unlink(path.c_str()); // Don't report this as a failure. An empty build log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_SUCCESS; } } @@ -340,7 +340,7 @@ bool BuildLog::Load(const string& path, string* err) { fclose(file); if (!line_start) { - return true; // file was empty + return LOAD_SUCCESS; // file was empty } // Decide whether it's time to rebuild the log: @@ -355,7 +355,7 @@ bool BuildLog::Load(const string& path, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { diff --git a/src/build_log.h b/src/build_log.h index d52dd3b..ed59d79 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -20,6 +20,7 @@ using namespace std; #include "hash_map.h" +#include "load_status.h" #include "timestamp.h" #include "util.h" // uint64_t @@ -50,7 +51,7 @@ struct BuildLog { void Close(); /// Load the on-disk log. - bool Load(const string& path, string* err); + LoadStatus Load(const string& path, string* err); struct LogEntry { string output; diff --git a/src/build_log_test.cc b/src/build_log_test.cc index eee8290..48ece23 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -151,7 +151,7 @@ TEST_F(BuildLogTest, Truncate) { BuildLog log3; err.clear(); - ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty()); + ASSERT_TRUE(log3.Load(kTestFilename, &err) == LOAD_SUCCESS || !err.empty()); } } diff --git a/src/deps_log.cc b/src/deps_log.cc index 4aaffeb..cf55194 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -167,15 +167,15 @@ void DepsLog::Close() { file_ = NULL; } -bool DepsLog::Load(const string& path, State* state, string* err) { +LoadStatus DepsLog::Load(const string& path, State* state, string* err) { METRIC_RECORD(".ninja_deps load"); char buf[kMaxRecordSize + 1]; FILE* f = fopen(path.c_str(), "rb"); if (!f) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } bool valid_header = true; @@ -196,7 +196,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { unlink(path.c_str()); // Don't report this as a failure. An empty deps log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_SUCCESS; } long offset; @@ -284,12 +284,12 @@ bool DepsLog::Load(const string& path, State* state, string* err) { fclose(f); if (!Truncate(path, offset, err)) - return false; + return LOAD_ERROR; // The truncate succeeded; we'll just report the load error as a // warning because the build can proceed. *err += "; recovering"; - return true; + return LOAD_SUCCESS; } fclose(f); @@ -302,7 +302,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } DepsLog::Deps* DepsLog::GetDeps(Node* node) { diff --git a/src/deps_log.h b/src/deps_log.h index 3812a28..e7974a1 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -21,6 +21,7 @@ using namespace std; #include +#include "load_status.h" #include "timestamp.h" struct Node; @@ -84,7 +85,7 @@ struct DepsLog { int node_count; Node** nodes; }; - bool Load(const string& path, State* state, string* err); + LoadStatus Load(const string& path, State* state, string* err); Deps* GetDeps(Node* node); /// Rewrite the known log entries, throwing away old data. diff --git a/src/load_status.h b/src/load_status.h new file mode 100644 index 0000000..0b16b1a --- /dev/null +++ b/src/load_status.h @@ -0,0 +1,24 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_LOAD_STATUS_H_ +#define NINJA_LOAD_STATUS_H_ + +enum LoadStatus { + LOAD_ERROR, + LOAD_SUCCESS, + LOAD_NOT_FOUND, +}; + +#endif // NINJA_LOAD_STATUS_H_ diff --git a/src/ninja.cc b/src/ninja.cc index 8b76ded..7fcb4f7 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -17,6 +17,7 @@ #include #include #include +#include #ifdef _WIN32 #include "getopt.h" @@ -130,11 +131,11 @@ struct NinjaMain : public BuildLogUser { int ToolRules(const Options* options, int argc, char* argv[]); /// Open the build log. - /// @return false on error. + /// @return LOAD_ERROR on error. bool OpenBuildLog(bool recompact_only = false); /// Open the deps log: load it, then open for writing. - /// @return false on error. + /// @return LOAD_ERROR on error. bool OpenDepsLog(bool recompact_only = false); /// Ensure the build directory exists, creating it if necessary. @@ -846,8 +847,8 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) { if (!EnsureBuildDirExists()) return 1; - if (!OpenBuildLog(/*recompact_only=*/true) || - !OpenDepsLog(/*recompact_only=*/true)) + if (OpenBuildLog(/*recompact_only=*/true) == LOAD_ERROR || + OpenDepsLog(/*recompact_only=*/true) == LOAD_ERROR) return 1; return 0; @@ -862,12 +863,17 @@ int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) { log_path = build_dir_ + "/" + log_path; string err; - if (!build_log_.Load(log_path, &err)) { + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return EXIT_FAILURE; } + if (status == LOAD_NOT_FOUND) { + // Nothing to restat, ignore this + return EXIT_SUCCESS; + } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } @@ -1068,17 +1074,21 @@ bool NinjaMain::OpenBuildLog(bool recompact_only) { log_path = build_dir_ + "/" + log_path; string err; - if (!build_log_.Load(log_path, &err)) { + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = build_log_.Recompact(log_path, *this, &err); if (!success) Error("failed recompaction: %s", err.c_str()); @@ -1103,17 +1113,21 @@ bool NinjaMain::OpenDepsLog(bool recompact_only) { path = build_dir_ + "/" + path; string err; - if (!deps_log_.Load(path, &state_, &err)) { + const LoadStatus status = deps_log_.Load(path, &state_, &err); + if (status == LOAD_ERROR) { Error("loading deps log %s: %s", path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = deps_log_.Recompact(path, &err); if (!success) Error("failed recompaction: %s", err.c_str()); -- cgit v0.12 From 66b746044d34c4791077488924f082b2fee7cb8a Mon Sep 17 00:00:00 2001 From: Jan Niklas Hasse Date: Mon, 23 Dec 2019 14:34:52 +0100 Subject: Add a short documentation for restat tool --- doc/manual.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index e49d26d..8f42efb 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -283,6 +283,9 @@ target, show just the target's dependencies. _Available since Ninja 1.4._ `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._ +`restat`:: updates all recorded file modification timestamps in the `.ninja_log` +file. _Available since Ninja 1.10._ + `rules`:: output the list of all rules (eventually with their description if they have one). It can be used to know which rule name to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. -- cgit v0.12