diff options
Diffstat (limited to 'src/deps_log.cc')
-rw-r--r-- | src/deps_log.cc | 326 |
1 files changed, 326 insertions, 0 deletions
diff --git a/src/deps_log.cc b/src/deps_log.cc new file mode 100644 index 0000000..931cc77 --- /dev/null +++ b/src/deps_log.cc @@ -0,0 +1,326 @@ +// Copyright 2012 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "deps_log.h" + +#include <assert.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#ifndef _WIN32 +#include <unistd.h> +#endif + +#include "graph.h" +#include "metrics.h" +#include "state.h" +#include "util.h" + +// The version is stored as 4 bytes after the signature and also serves as a +// byte order mark. Signature and version combined are 16 bytes long. +const char kFileSignature[] = "# ninjadeps\n"; +const int kCurrentVersion = 1; + +DepsLog::~DepsLog() { + Close(); +} + +bool DepsLog::OpenForWrite(const string& path, string* err) { + if (needs_recompaction_) { + Close(); + if (!Recompact(path, err)) + return false; + } + + file_ = fopen(path.c_str(), "ab"); + if (!file_) { + *err = strerror(errno); + return false; + } + SetCloseOnExec(fileno(file_)); + + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(file_, 0, SEEK_END); + + if (ftell(file_) == 0) { + if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) { + *err = strerror(errno); + return false; + } + if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) { + *err = strerror(errno); + return false; + } + } + + return true; +} + +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, + const vector<Node*>& nodes) { + return RecordDeps(node, mtime, nodes.size(), + nodes.empty() ? NULL : (Node**)&nodes.front()); +} + +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, + int node_count, Node** nodes) { + // Track whether there's any new data to be recorded. + bool made_change = false; + + // Assign ids to all nodes that are missing one. + if (node->id() < 0) { + RecordId(node); + made_change = true; + } + for (int i = 0; i < node_count; ++i) { + if (nodes[i]->id() < 0) { + RecordId(nodes[i]); + made_change = true; + } + } + + // See if the new data is different than the existing data, if any. + if (!made_change) { + Deps* deps = GetDeps(node); + if (!deps || + deps->mtime != mtime || + deps->node_count != node_count) { + made_change = true; + } else { + for (int i = 0; i < node_count; ++i) { + if (deps->nodes[i] != nodes[i]) { + made_change = true; + break; + } + } + } + } + + // Don't write anything if there's no new info. + if (!made_change) + return true; + + // Update on-disk representation. + uint16_t size = 4 * (1 + 1 + (uint16_t)node_count); + size |= 0x8000; // Deps record: set high bit. + fwrite(&size, 2, 1, file_); + int id = node->id(); + fwrite(&id, 4, 1, file_); + int timestamp = mtime; + fwrite(×tamp, 4, 1, file_); + for (int i = 0; i < node_count; ++i) { + id = nodes[i]->id(); + fwrite(&id, 4, 1, file_); + } + + // Update in-memory representation. + Deps* deps = new Deps(mtime, node_count); + for (int i = 0; i < node_count; ++i) + deps->nodes[i] = nodes[i]; + UpdateDeps(node->id(), deps); + + return true; +} + +void DepsLog::Close() { + if (file_) + fclose(file_); + file_ = NULL; +} + +bool DepsLog::Load(const string& path, State* state, string* err) { + METRIC_RECORD(".ninja_deps load"); + char buf[32 << 10]; + FILE* f = fopen(path.c_str(), "rb"); + if (!f) { + if (errno == ENOENT) + return true; + *err = strerror(errno); + return false; + } + + bool valid_header = true; + int version = 0; + if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1) + valid_header = false; + if (!valid_header || strcmp(buf, kFileSignature) != 0 || + version != kCurrentVersion) { + *err = "bad deps log signature or version; starting over"; + fclose(f); + unlink(path.c_str()); + // Don't report this as a failure. An empty deps log will cause + // us to rebuild the outputs anyway. + return true; + } + + long offset; + bool read_failed = false; + int unique_dep_record_count = 0; + int total_dep_record_count = 0; + for (;;) { + offset = ftell(f); + + uint16_t size; + if (fread(&size, 2, 1, f) < 1) { + if (!feof(f)) + read_failed = true; + break; + } + bool is_deps = (size >> 15) != 0; + size = size & 0x7FFF; + + if (fread(buf, size, 1, f) < 1) { + read_failed = true; + break; + } + + if (is_deps) { + assert(size % 4 == 0); + int* deps_data = reinterpret_cast<int*>(buf); + int out_id = deps_data[0]; + int mtime = deps_data[1]; + deps_data += 2; + int deps_count = (size / 4) - 2; + + Deps* deps = new Deps(mtime, deps_count); + for (int i = 0; i < deps_count; ++i) { + assert(deps_data[i] < (int)nodes_.size()); + assert(nodes_[deps_data[i]]); + deps->nodes[i] = nodes_[deps_data[i]]; + } + + total_dep_record_count++; + if (!UpdateDeps(out_id, deps)) + ++unique_dep_record_count; + } else { + StringPiece path(buf, size); + Node* node = state->GetNode(path); + assert(node->id() < 0); + node->set_id(nodes_.size()); + nodes_.push_back(node); + } + } + + if (read_failed) { + // An error occurred while loading; try to recover by truncating the + // file to the last fully-read record. + if (ferror(f)) { + *err = strerror(ferror(f)); + } else { + *err = "premature end of file"; + } + fclose(f); + + if (!Truncate(path.c_str(), offset, err)) + return false; + + // The truncate succeeded; we'll just report the load error as a + // warning because the build can proceed. + *err += "; recovering"; + return true; + } + + fclose(f); + + // Rebuild the log if there are too many dead records. + int kMinCompactionEntryCount = 1000; + int kCompactionRatio = 3; + if (total_dep_record_count > kMinCompactionEntryCount && + total_dep_record_count > unique_dep_record_count * kCompactionRatio) { + needs_recompaction_ = true; + } + + return true; +} + +DepsLog::Deps* DepsLog::GetDeps(Node* node) { + // Abort if the node has no id (never referenced in the deps) or if + // there's no deps recorded for the node. + if (node->id() < 0 || node->id() >= (int)deps_.size()) + return NULL; + return deps_[node->id()]; +} + +bool DepsLog::Recompact(const string& path, string* err) { + METRIC_RECORD(".ninja_deps recompact"); + printf("Recompacting deps...\n"); + + string temp_path = path + ".recompact"; + + // OpenForWrite() opens for append. Make sure it's not appending to a + // left-over file from a previous recompaction attempt that crashed somehow. + unlink(temp_path.c_str()); + + DepsLog new_log; + if (!new_log.OpenForWrite(temp_path, err)) + return false; + + // Clear all known ids so that new ones can be reassigned. The new indices + // will refer to the ordering in new_log, not in the current log. + for (vector<Node*>::iterator i = nodes_.begin(); i != nodes_.end(); ++i) + (*i)->set_id(-1); + + // Write out all deps again. + for (int old_id = 0; old_id < (int)deps_.size(); ++old_id) { + Deps* deps = deps_[old_id]; + if (!deps) continue; // If nodes_[old_id] is a leaf, it has no deps. + + if (!new_log.RecordDeps(nodes_[old_id], deps->mtime, + deps->node_count, deps->nodes)) { + new_log.Close(); + return false; + } + } + + new_log.Close(); + + // All nodes now have ids that refer to new_log, so steal its data. + deps_.swap(new_log.deps_); + nodes_.swap(new_log.nodes_); + + if (unlink(path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.c_str()) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} + +bool DepsLog::UpdateDeps(int out_id, Deps* deps) { + if (out_id >= (int)deps_.size()) + deps_.resize(out_id + 1); + + bool delete_old = deps_[out_id] != NULL; + if (delete_old) + delete deps_[out_id]; + deps_[out_id] = deps; + return delete_old; +} + +bool DepsLog::RecordId(Node* node) { + uint16_t size = (uint16_t)node->path().size(); + fwrite(&size, 2, 1, file_); + fwrite(node->path().data(), node->path().size(), 1, file_); + + node->set_id(nodes_.size()); + nodes_.push_back(node); + + return true; +} |