From 945a1080afc7f79d50e166428af754a3e5d5d91c Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 24 May 2014 21:17:35 -0700
Subject: Add a stat cache. Demo-quality, and disabled atm.

---
 src/disk_interface.cc | 119 ++++++++++++++++++++++++++++++++++++++++++--------
 src/disk_interface.h  |   9 +++-
 src/ninja.cc          |   6 +++
 3 files changed, 114 insertions(+), 20 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 4dfae1a..2175051 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -55,6 +55,29 @@ int MakeDir(const string& path) {
 #endif
 }
 
+TimeStamp StatSingleFile(const string& path, bool quiet) {
+  WIN32_FILE_ATTRIBUTE_DATA attrs;
+  if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
+    DWORD err = GetLastError();
+    if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
+      return 0;
+    if (!quiet) {
+      Error("GetFileAttributesEx(%s): %s", path.c_str(),
+            GetLastErrorString().c_str());
+    }
+    return -1;
+  }
+  const FILETIME& filetime = attrs.ftLastWriteTime;
+  // FILETIME is in 100-nanosecond increments since the Windows epoch.
+  // We don't much care about epoch correctness but we do want the
+  // resulting value to fit in an integer.
+  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
+    ((uint64_t)filetime.dwLowDateTime);
+  mtime /= 1000000000LL / 100; // 100ns -> s.
+  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
+  return (TimeStamp)mtime;
+}
+
 }  // namespace
 
 // DiskInterface ---------------------------------------------------------------
@@ -89,26 +112,84 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
     }
     return -1;
   }
-  WIN32_FILE_ATTRIBUTE_DATA attrs;
-  if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
-    DWORD err = GetLastError();
-    if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
-      return 0;
-    if (!quiet_) {
-      Error("GetFileAttributesEx(%s): %s", path.c_str(),
-            GetLastErrorString().c_str());
-    }
-    return -1;
+  if (!use_cache_)
+    return StatSingleFile(path, quiet_);
+
+  string dir = DirName(path);
+  int offs = dir.size();
+  if (offs) ++offs;  // skip \ too
+  string base(path.substr(offs));
+
+  std::transform(dir.begin(), dir.end(), dir.begin(), ::tolower);
+  std::transform(base.begin(), base.end(), base.begin(), ::tolower);
+
+  Cache::iterator ci = cache_.find(dir);
+  if (ci != cache_.end()) {
+    DirCache::iterator di = ci->second->find(base);
+    if (di != ci->second->end())
+      return di->second;
+    return 0;
   }
-  const FILETIME& filetime = attrs.ftLastWriteTime;
-  // FILETIME is in 100-nanosecond increments since the Windows epoch.
-  // We don't much care about epoch correctness but we do want the
-  // resulting value to fit in an integer.
-  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
-    ((uint64_t)filetime.dwLowDateTime);
-  mtime /= 1000000000LL / 100; // 100ns -> s.
-  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
-  return (TimeStamp)mtime;
+
+  if (dir.empty())
+    dir = ".";
+  // XXX fill in dir using FFF / FNF
+  HANDLE hFind = INVALID_HANDLE_VALUE;
+  WIN32_FIND_DATAA ffd;
+
+#if 0
+  hFind = FindFirstFileA((dir + "\\*").c_str(), &ffd);
+#else
+  hFind = FindFirstFileExA((dir + "\\*").c_str(),
+                 //FindExInfoStandard,
+                 FindExInfoBasic,  // 30% faster than FindExInfoStandard!
+                 &ffd, 
+                 FindExSearchNameMatch, 
+                 NULL, 
+                 0 );  // XXX: check FIND_FIRST_EX_LARGE_FETCH
+#endif
+
+  if (hFind == INVALID_HANDLE_VALUE) {
+    fprintf(stderr, "fail %s", dir.c_str());
+    exit(-1);
+  }
+  DirCache* dc = new DirCache;
+  do {
+    if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+      continue;
+
+    string lowername = ffd.cFileName;
+    std::transform(lowername.begin(), lowername.end(),
+                   lowername.begin(), ::tolower);
+
+//fprintf(stderr, "%s %s\n", dir.c_str(), lowername.c_str());
+
+    const FILETIME& filetime = ffd.ftLastWriteTime;
+    // FILETIME is in 100-nanosecond increments since the Windows epoch.
+    // We don't much care about epoch correctness but we do want the
+    // resulting value to fit in an integer.
+    uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
+        ((uint64_t)filetime.dwLowDateTime);
+    mtime /= 1000000000LL / 100; // 100ns -> s.
+//if(mtime == 0)
+//printf("   asdasdfadsfsadf\n");
+
+    mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
+
+    (*dc).insert(make_pair(lowername, (TimeStamp)mtime));
+  } while (FindNextFileA(hFind, &ffd));
+  FindClose(hFind);
+
+  if (dir == ".")
+    cache_.insert(make_pair("", dc));
+  else
+    cache_.insert(make_pair(dir, dc));
+
+  DirCache::iterator di = dc->find(base);
+  if (di != dc->end())
+    return di->second;
+  return 0;
+
 #else
   struct stat st;
   if (stat(path.c_str(), &st) < 0) {
diff --git a/src/disk_interface.h b/src/disk_interface.h
index ff1e21c..d368c38 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -15,6 +15,7 @@
 #ifndef NINJA_DISK_INTERFACE_H_
 #define NINJA_DISK_INTERFACE_H_
 
+#include <map>
 #include <string>
 using namespace std;
 
@@ -55,7 +56,7 @@ struct DiskInterface {
 
 /// Implementation of DiskInterface that actually hits the disk.
 struct RealDiskInterface : public DiskInterface {
-  RealDiskInterface() : quiet_(false) {}
+  RealDiskInterface() : quiet_(false), use_cache_(false) {}
   virtual ~RealDiskInterface() {}
   virtual TimeStamp Stat(const string& path);
   virtual bool MakeDir(const string& path);
@@ -65,6 +66,12 @@ struct RealDiskInterface : public DiskInterface {
 
   /// Whether to print on errors.  Used to make a test quieter.
   bool quiet_;
+  /// Whether stat information can be cached.
+  bool use_cache_;
+
+  typedef map<string, TimeStamp> DirCache;
+  typedef map<string, DirCache*> Cache;
+  Cache cache_;
 };
 
 #endif  // NINJA_DISK_INTERFACE_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index 4c8dab7..962480e 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -882,6 +882,9 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     return 1;
   }
 
+// XXX allow stat caching
+  //disk_interface_.use_cache_ = true;
+
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
     if (!builder.AddTarget(targets[i], &err)) {
@@ -895,6 +898,9 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     }
   }
 
+// XXX disallow stat caching
+  disk_interface_.use_cache_ = false;
+
   if (builder.AlreadyUpToDate()) {
     printf("ninja: no work to do.\n");
     return 0;
-- 
cgit v0.12


From ef73a646ea32eb243c9571d5f7a5f386bb1461b7 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 24 May 2014 21:36:57 -0700
Subject: Turn on stat cache. Empty builds of chrome on my laptop 4s -> 1.3s
 (!)

---
 src/ninja.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ninja.cc b/src/ninja.cc
index 962480e..a2f3111 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -883,7 +883,7 @@ int NinjaMain::RunBuild(int argc, char** argv) {
   }
 
 // XXX allow stat caching
-  //disk_interface_.use_cache_ = true;
+  disk_interface_.use_cache_ = true;
 
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
-- 
cgit v0.12


From 238b259fd7f0361aa4cd2037b4695cbb668d9f7c Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 21:29:18 -0700
Subject: minor cleanups

---
 src/disk_interface.cc | 96 +++++++++++++++++++++------------------------------
 1 file changed, 40 insertions(+), 56 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 2175051..b4a00b4 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -55,6 +55,17 @@ int MakeDir(const string& path) {
 #endif
 }
 
+TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
+  // FILETIME is in 100-nanosecond increments since the Windows epoch.
+  // We don't much care about epoch correctness but we do want the
+  // resulting value to fit in an integer.
+  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
+    ((uint64_t)filetime.dwLowDateTime);
+  mtime /= 1000000000LL / 100; // 100ns -> s.
+  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
+  return (TimeStamp)mtime;
+}
+
 TimeStamp StatSingleFile(const string& path, bool quiet) {
   WIN32_FILE_ATTRIBUTE_DATA attrs;
   if (!GetFileAttributesEx(path.c_str(), GetFileExInfoStandard, &attrs)) {
@@ -67,15 +78,32 @@ TimeStamp StatSingleFile(const string& path, bool quiet) {
     }
     return -1;
   }
-  const FILETIME& filetime = attrs.ftLastWriteTime;
-  // FILETIME is in 100-nanosecond increments since the Windows epoch.
-  // We don't much care about epoch correctness but we do want the
-  // resulting value to fit in an integer.
-  uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
-    ((uint64_t)filetime.dwLowDateTime);
-  mtime /= 1000000000LL / 100; // 100ns -> s.
-  mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
-  return (TimeStamp)mtime;
+  return TimeStampFromFileTime(attrs.ftLastWriteTime);
+}
+
+void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps) {
+  HANDLE hFind = INVALID_HANDLE_VALUE;
+  WIN32_FIND_DATAA ffd;
+
+  // FindExInfoBasic is 30% faster than FindExInfoStandard.
+  hFind = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic, &ffd,
+                           FindExSearchNameMatch, NULL, 0);
+
+  if (hFind == INVALID_HANDLE_VALUE) {
+    fprintf(stderr, "fail %s", dir.c_str());
+    exit(-1);
+  }
+  do {
+    if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+      continue;
+
+    string lowername = ffd.cFileName;
+    transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower);
+
+    const FILETIME& filetime = ffd.ftLastWriteTime;
+    (*stamps).insert(make_pair(lowername, TimeStampFromFileTime(filetime)));
+  } while (FindNextFileA(hFind, &ffd));
+  FindClose(hFind);
 }
 
 }  // namespace
@@ -120,8 +148,8 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
   if (offs) ++offs;  // skip \ too
   string base(path.substr(offs));
 
-  std::transform(dir.begin(), dir.end(), dir.begin(), ::tolower);
-  std::transform(base.begin(), base.end(), base.begin(), ::tolower);
+  transform(dir.begin(), dir.end(), dir.begin(), ::tolower);
+  transform(base.begin(), base.end(), base.begin(), ::tolower);
 
   Cache::iterator ci = cache_.find(dir);
   if (ci != cache_.end()) {
@@ -133,52 +161,8 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
 
   if (dir.empty())
     dir = ".";
-  // XXX fill in dir using FFF / FNF
-  HANDLE hFind = INVALID_HANDLE_VALUE;
-  WIN32_FIND_DATAA ffd;
-
-#if 0
-  hFind = FindFirstFileA((dir + "\\*").c_str(), &ffd);
-#else
-  hFind = FindFirstFileExA((dir + "\\*").c_str(),
-                 //FindExInfoStandard,
-                 FindExInfoBasic,  // 30% faster than FindExInfoStandard!
-                 &ffd, 
-                 FindExSearchNameMatch, 
-                 NULL, 
-                 0 );  // XXX: check FIND_FIRST_EX_LARGE_FETCH
-#endif
-
-  if (hFind == INVALID_HANDLE_VALUE) {
-    fprintf(stderr, "fail %s", dir.c_str());
-    exit(-1);
-  }
   DirCache* dc = new DirCache;
-  do {
-    if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
-      continue;
-
-    string lowername = ffd.cFileName;
-    std::transform(lowername.begin(), lowername.end(),
-                   lowername.begin(), ::tolower);
-
-//fprintf(stderr, "%s %s\n", dir.c_str(), lowername.c_str());
-
-    const FILETIME& filetime = ffd.ftLastWriteTime;
-    // FILETIME is in 100-nanosecond increments since the Windows epoch.
-    // We don't much care about epoch correctness but we do want the
-    // resulting value to fit in an integer.
-    uint64_t mtime = ((uint64_t)filetime.dwHighDateTime << 32) |
-        ((uint64_t)filetime.dwLowDateTime);
-    mtime /= 1000000000LL / 100; // 100ns -> s.
-//if(mtime == 0)
-//printf("   asdasdfadsfsadf\n");
-
-    mtime -= 12622770400LL;  // 1600 epoch -> 2000 epoch (subtract 400 years).
-
-    (*dc).insert(make_pair(lowername, (TimeStamp)mtime));
-  } while (FindNextFileA(hFind, &ffd));
-  FindClose(hFind);
+  StatAllFilesInDir(dir, dc);
 
   if (dir == ".")
     cache_.insert(make_pair("", dc));
-- 
cgit v0.12


From 20e03b33a340bbc709b0ecd8c50a4b55e709322b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 21:36:30 -0700
Subject: more minor cleanups

---
 src/disk_interface.cc | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index b4a00b4..f211698 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -82,12 +82,10 @@ TimeStamp StatSingleFile(const string& path, bool quiet) {
 }
 
 void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps) {
-  HANDLE hFind = INVALID_HANDLE_VALUE;
-  WIN32_FIND_DATAA ffd;
-
   // FindExInfoBasic is 30% faster than FindExInfoStandard.
-  hFind = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic, &ffd,
-                           FindExSearchNameMatch, NULL, 0);
+  WIN32_FIND_DATAA ffd;
+  HANDLE hFind = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic, &ffd,
+                                  FindExSearchNameMatch, NULL, 0);
 
   if (hFind == INVALID_HANDLE_VALUE) {
     fprintf(stderr, "fail %s", dir.c_str());
@@ -154,26 +152,15 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
   Cache::iterator ci = cache_.find(dir);
   if (ci != cache_.end()) {
     DirCache::iterator di = ci->second->find(base);
-    if (di != ci->second->end())
-      return di->second;
-    return 0;
+    return di != ci->second->end() ? di->second : 0;
   }
 
-  if (dir.empty())
-    dir = ".";
   DirCache* dc = new DirCache;
-  StatAllFilesInDir(dir, dc);
-
-  if (dir == ".")
-    cache_.insert(make_pair("", dc));
-  else
-    cache_.insert(make_pair(dir, dc));
+  StatAllFilesInDir(dir.empty() ? "." : dir, dc);
+  cache_.insert(make_pair(dir, dc));
 
   DirCache::iterator di = dc->find(base);
-  if (di != dc->end())
-    return di->second;
-  return 0;
-
+  return di != dc->end() ? di->second : 0;
 #else
   struct stat st;
   if (stat(path.c_str(), &st) < 0) {
-- 
cgit v0.12


From f5f7c80897cbe2d7dca454283e4e9c5e484dfadb Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 21:38:44 -0700
Subject: more minor cleanups

---
 src/disk_interface.cc | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index f211698..784f5d5 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -150,17 +150,13 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
   transform(base.begin(), base.end(), base.begin(), ::tolower);
 
   Cache::iterator ci = cache_.find(dir);
-  if (ci != cache_.end()) {
-    DirCache::iterator di = ci->second->find(base);
-    return di != ci->second->end() ? di->second : 0;
+  if (ci == cache_.end()) {
+    DirCache* dc = new DirCache;
+    StatAllFilesInDir(dir.empty() ? "." : dir, dc);
+    ci = cache_.insert(make_pair(dir, dc)).first;
   }
-
-  DirCache* dc = new DirCache;
-  StatAllFilesInDir(dir.empty() ? "." : dir, dc);
-  cache_.insert(make_pair(dir, dc));
-
-  DirCache::iterator di = dc->find(base);
-  return di != dc->end() ? di->second : 0;
+  DirCache::iterator di = ci->second->find(base);
+  return di != ci->second->end() ? di->second : 0;
 #else
   struct stat st;
   if (stat(path.c_str(), &st) < 0) {
-- 
cgit v0.12


From d64f672e5083e01c579191b02c1e1667d9e63ef2 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 21:50:05 -0700
Subject: simplify more, move behind flag

---
 src/debug_flags.cc    |  2 ++
 src/debug_flags.h     |  4 ++++
 src/disk_interface.cc |  4 +---
 src/ninja.cc          | 15 +++++++++++----
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/src/debug_flags.cc b/src/debug_flags.cc
index 75f1ea5..786a100 100644
--- a/src/debug_flags.cc
+++ b/src/debug_flags.cc
@@ -15,3 +15,5 @@
 bool g_explaining = false;
 
 bool g_keep_rsp = false;
+
+bool g_experimental_win_statcache = false;
diff --git a/src/debug_flags.h b/src/debug_flags.h
index ba3ebf3..ce3c292 100644
--- a/src/debug_flags.h
+++ b/src/debug_flags.h
@@ -26,4 +26,8 @@ extern bool g_explaining;
 
 extern bool g_keep_rsp;
 
+#ifdef _WIN32
+extern bool g_experimental_win_statcache;
+#endif  // _WIN32
+
 #endif // NINJA_EXPLAIN_H_
diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 784f5d5..b7943ef 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -142,9 +142,7 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
     return StatSingleFile(path, quiet_);
 
   string dir = DirName(path);
-  int offs = dir.size();
-  if (offs) ++offs;  // skip \ too
-  string base(path.substr(offs));
+  string base(path.substr(dir.size() ? dir.size() + 1 : 0));
 
   transform(dir.begin(), dir.end(), dir.begin(), ::tolower);
   transform(base.begin(), base.end(), base.begin(), ::tolower);
diff --git a/src/ninja.cc b/src/ninja.cc
index a2f3111..acb793e 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -760,6 +760,9 @@ bool DebugEnable(const string& name) {
 "  stats    print operation counts/timing info\n"
 "  explain  explain what caused a command to execute\n"
 "  keeprsp  don't delete @response files on success\n"
+#ifdef _WIN32
+"  experimentalwinstatcache  use an alternative method for stat()ing files\n"
+#endif
 "multiple modes can be enabled via -d FOO -d BAR\n");
     return false;
   } else if (name == "stats") {
@@ -771,9 +774,13 @@ bool DebugEnable(const string& name) {
   } else if (name == "keeprsp") {
     g_keep_rsp = true;
     return true;
+  } else if (name == "experimentalwinstatcache") {
+    g_experimental_win_statcache = true;
+    return true;
   } else {
     const char* suggestion =
-        SpellcheckString(name.c_str(), "stats", "explain", "keeprsp", NULL);
+        SpellcheckString(name.c_str(), "stats", "explain", "keeprsp",
+        "experimentalwinstatcache", NULL);
     if (suggestion) {
       Error("unknown debug setting '%s', did you mean '%s'?",
             name.c_str(), suggestion);
@@ -882,8 +889,8 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     return 1;
   }
 
-// XXX allow stat caching
-  disk_interface_.use_cache_ = true;
+  if (g_experimental_win_statcache)
+    disk_interface_.use_cache_ = true;
 
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
@@ -898,7 +905,7 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     }
   }
 
-// XXX disallow stat caching
+  // Make sure restat rules do not see stale timestamps.
   disk_interface_.use_cache_ = false;
 
   if (builder.AlreadyUpToDate()) {
-- 
cgit v0.12


From f6f63eb50855ac2ccf37e2f5629f6a9e962d6c5e Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 22:05:09 -0700
Subject: make win-only

---
 src/disk_interface.h | 2 ++
 src/ninja.cc         | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/src/disk_interface.h b/src/disk_interface.h
index d368c38..bffb679 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -66,12 +66,14 @@ struct RealDiskInterface : public DiskInterface {
 
   /// Whether to print on errors.  Used to make a test quieter.
   bool quiet_;
+#ifdef _WIN32
   /// Whether stat information can be cached.
   bool use_cache_;
 
   typedef map<string, TimeStamp> DirCache;
   typedef map<string, DirCache*> Cache;
   Cache cache_;
+#endif
 };
 
 #endif  // NINJA_DISK_INTERFACE_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index acb793e..ccfc14d 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -889,8 +889,10 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     return 1;
   }
 
+#ifdef _WIN32
   if (g_experimental_win_statcache)
     disk_interface_.use_cache_ = true;
+#endif
 
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
@@ -905,8 +907,10 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     }
   }
 
+#ifdef _WIN32
   // Make sure restat rules do not see stale timestamps.
   disk_interface_.use_cache_ = false;
+#endif
 
   if (builder.AlreadyUpToDate()) {
     printf("ninja: no work to do.\n");
-- 
cgit v0.12


From d902b5f24d4c56aa2fc82b34246b0f75766f10f8 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 22:07:04 -0700
Subject: comment

---
 src/disk_interface.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/disk_interface.h b/src/disk_interface.h
index bffb679..bb40dc9 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -56,7 +56,11 @@ struct DiskInterface {
 
 /// Implementation of DiskInterface that actually hits the disk.
 struct RealDiskInterface : public DiskInterface {
-  RealDiskInterface() : quiet_(false), use_cache_(false) {}
+  RealDiskInterface() : quiet_(false)
+#ifdef _WIN32
+                      , use_cache_(false)
+#endif
+                      {}
   virtual ~RealDiskInterface() {}
   virtual TimeStamp Stat(const string& path);
   virtual bool MakeDir(const string& path);
@@ -71,6 +75,8 @@ struct RealDiskInterface : public DiskInterface {
   bool use_cache_;
 
   typedef map<string, TimeStamp> DirCache;
+  // TODO: Neither a map nor a hashmap seems ideal here.  If the statcache
+  // works out, come up with a better data structure.
   typedef map<string, DirCache*> Cache;
   Cache cache_;
 #endif
-- 
cgit v0.12


From f9c23f8fad43997a08b7e2f7e4c56cc9d9bcb5e1 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 22:11:31 -0700
Subject: on by default

---
 src/debug_flags.cc | 2 +-
 src/ninja.cc       | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/debug_flags.cc b/src/debug_flags.cc
index 786a100..ccd4396 100644
--- a/src/debug_flags.cc
+++ b/src/debug_flags.cc
@@ -16,4 +16,4 @@ bool g_explaining = false;
 
 bool g_keep_rsp = false;
 
-bool g_experimental_win_statcache = false;
+bool g_experimental_win_statcache = true;
diff --git a/src/ninja.cc b/src/ninja.cc
index ccfc14d..cace2a0 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -761,7 +761,7 @@ bool DebugEnable(const string& name) {
 "  explain  explain what caused a command to execute\n"
 "  keeprsp  don't delete @response files on success\n"
 #ifdef _WIN32
-"  experimentalwinstatcache  use an alternative method for stat()ing files\n"
+"  nowinstatcache  don't batch stat() calls per directory and cache them\n"
 #endif
 "multiple modes can be enabled via -d FOO -d BAR\n");
     return false;
@@ -774,13 +774,13 @@ bool DebugEnable(const string& name) {
   } else if (name == "keeprsp") {
     g_keep_rsp = true;
     return true;
-  } else if (name == "experimentalwinstatcache") {
-    g_experimental_win_statcache = true;
+  } else if (name == "nowinstatcache") {
+    g_experimental_win_statcache = false;
     return true;
   } else {
     const char* suggestion =
         SpellcheckString(name.c_str(), "stats", "explain", "keeprsp",
-        "experimentalwinstatcache", NULL);
+        "nowinstatcache", NULL);
     if (suggestion) {
       Error("unknown debug setting '%s', did you mean '%s'?",
             name.c_str(), suggestion);
-- 
cgit v0.12


From 9970174e2ab8e4d5c9f333b795c8d284cf1cb8f5 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sat, 14 Jun 2014 22:18:28 -0700
Subject: error checking

---
 src/disk_interface.cc | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index b7943ef..cd99915 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -81,25 +81,28 @@ TimeStamp StatSingleFile(const string& path, bool quiet) {
   return TimeStampFromFileTime(attrs.ftLastWriteTime);
 }
 
-void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps) {
+void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
+                       bool quiet) {
   // FindExInfoBasic is 30% faster than FindExInfoStandard.
   WIN32_FIND_DATAA ffd;
   HANDLE hFind = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic, &ffd,
                                   FindExSearchNameMatch, NULL, 0);
 
   if (hFind == INVALID_HANDLE_VALUE) {
-    fprintf(stderr, "fail %s", dir.c_str());
-    exit(-1);
+    DWORD err = GetLastError();
+    if (err != ERROR_FILE_NOT_FOUND && err != ERROR_PATH_NOT_FOUND && !quiet) {
+      Error("FindFirstFileExA(%s): %s", dir.c_str(),
+            GetLastErrorString().c_str());
+    }
+    return;
   }
   do {
     if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
       continue;
-
     string lowername = ffd.cFileName;
     transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower);
-
-    const FILETIME& filetime = ffd.ftLastWriteTime;
-    (*stamps).insert(make_pair(lowername, TimeStampFromFileTime(filetime)));
+    stamps->insert(make_pair(lowername,
+                             TimeStampFromFileTime(ffd.ftLastWriteTime)));
   } while (FindNextFileA(hFind, &ffd));
   FindClose(hFind);
 }
@@ -150,7 +153,7 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
   Cache::iterator ci = cache_.find(dir);
   if (ci == cache_.end()) {
     DirCache* dc = new DirCache;
-    StatAllFilesInDir(dir.empty() ? "." : dir, dc);
+    StatAllFilesInDir(dir.empty() ? "." : dir, dc, quiet_);
     ci = cache_.insert(make_pair(dir, dc)).first;
   }
   DirCache::iterator di = ci->second->find(base);
-- 
cgit v0.12


From 726afc8226a10cd6c5ce724a845ff5cd17169091 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sun, 15 Jun 2014 14:11:27 -0700
Subject: Free cache memory once it's no longer used.

Doesn't slow down empty build times measurably, and
saves some memory on non-empty builds.
---
 src/disk_interface.cc | 16 ++++++++++++++++
 src/disk_interface.h  |  6 +++++-
 src/ninja.cc          |  9 ++-------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index cd99915..ff86ed1 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -230,3 +230,19 @@ int RealDiskInterface::RemoveFile(const string& path) {
     return 0;
   }
 }
+
+void RealDiskInterface::AllowCache(bool allow) {
+#ifdef _WIN32
+  use_cache_ = allow;
+  if (!use_cache_)
+    ClearCache();
+#endif
+}
+
+void RealDiskInterface::ClearCache() {
+#ifdef _WIN32
+  for (Cache::iterator it = cache_.begin(), end = cache_.end(); it != end; ++it)
+    delete it->second;
+  cache_.clear();
+#endif
+}
diff --git a/src/disk_interface.h b/src/disk_interface.h
index bb40dc9..40c24b6 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -61,7 +61,7 @@ struct RealDiskInterface : public DiskInterface {
                       , use_cache_(false)
 #endif
                       {}
-  virtual ~RealDiskInterface() {}
+  virtual ~RealDiskInterface() { ClearCache(); }
   virtual TimeStamp Stat(const string& path);
   virtual bool MakeDir(const string& path);
   virtual bool WriteFile(const string& path, const string& contents);
@@ -70,6 +70,9 @@ struct RealDiskInterface : public DiskInterface {
 
   /// Whether to print on errors.  Used to make a test quieter.
   bool quiet_;
+
+  /// Whether stat information can be cached.
+  void AllowCache(bool allow);
 #ifdef _WIN32
   /// Whether stat information can be cached.
   bool use_cache_;
@@ -80,6 +83,7 @@ struct RealDiskInterface : public DiskInterface {
   typedef map<string, DirCache*> Cache;
   Cache cache_;
 #endif
+  void ClearCache();
 };
 
 #endif  // NINJA_DISK_INTERFACE_H_
diff --git a/src/ninja.cc b/src/ninja.cc
index cace2a0..eedfec0 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -889,10 +889,7 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     return 1;
   }
 
-#ifdef _WIN32
-  if (g_experimental_win_statcache)
-    disk_interface_.use_cache_ = true;
-#endif
+  disk_interface_.AllowCache(g_experimental_win_statcache);
 
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
@@ -907,10 +904,8 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     }
   }
 
-#ifdef _WIN32
   // Make sure restat rules do not see stale timestamps.
-  disk_interface_.use_cache_ = false;
-#endif
+  disk_interface_.AllowCache(false);
 
   if (builder.AlreadyUpToDate()) {
     printf("ninja: no work to do.\n");
-- 
cgit v0.12


From 4eb8309251c4839de25502a5390270b53d9706eb Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sun, 15 Jun 2014 14:42:42 -0700
Subject: add some statcache tests

---
 src/disk_interface.cc      | 16 +++++++++++-----
 src/disk_interface.h       |  4 +++-
 src/disk_interface_test.cc | 27 +++++++++++++++++++++++++++
 src/ninja.cc               |  4 ++--
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index ff86ed1..c4531e6 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -81,7 +81,7 @@ TimeStamp StatSingleFile(const string& path, bool quiet) {
   return TimeStampFromFileTime(attrs.ftLastWriteTime);
 }
 
-void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
+bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
                        bool quiet) {
   // FindExInfoBasic is 30% faster than FindExInfoStandard.
   WIN32_FIND_DATAA ffd;
@@ -90,11 +90,13 @@ void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
 
   if (hFind == INVALID_HANDLE_VALUE) {
     DWORD err = GetLastError();
-    if (err != ERROR_FILE_NOT_FOUND && err != ERROR_PATH_NOT_FOUND && !quiet) {
+    if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
+      return true;
+    if (!quiet) {
       Error("FindFirstFileExA(%s): %s", dir.c_str(),
             GetLastErrorString().c_str());
     }
-    return;
+    return false;
   }
   do {
     if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
@@ -105,6 +107,7 @@ void StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
                              TimeStampFromFileTime(ffd.ftLastWriteTime)));
   } while (FindNextFileA(hFind, &ffd));
   FindClose(hFind);
+  return true;
 }
 
 }  // namespace
@@ -153,7 +156,10 @@ TimeStamp RealDiskInterface::Stat(const string& path) {
   Cache::iterator ci = cache_.find(dir);
   if (ci == cache_.end()) {
     DirCache* dc = new DirCache;
-    StatAllFilesInDir(dir.empty() ? "." : dir, dc, quiet_);
+    if (!StatAllFilesInDir(dir.empty() ? "." : dir, dc, quiet_)) {
+      delete dc;
+      return -1;
+    }
     ci = cache_.insert(make_pair(dir, dc)).first;
   }
   DirCache::iterator di = ci->second->find(base);
@@ -231,7 +237,7 @@ int RealDiskInterface::RemoveFile(const string& path) {
   }
 }
 
-void RealDiskInterface::AllowCache(bool allow) {
+void RealDiskInterface::AllowStatCache(bool allow) {
 #ifdef _WIN32
   use_cache_ = allow;
   if (!use_cache_)
diff --git a/src/disk_interface.h b/src/disk_interface.h
index 40c24b6..a8a3023 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -72,7 +72,9 @@ struct RealDiskInterface : public DiskInterface {
   bool quiet_;
 
   /// Whether stat information can be cached.
-  void AllowCache(bool allow);
+  void AllowStatCache(bool allow);
+
+ private:
 #ifdef _WIN32
   /// Whether stat information can be cached.
   bool use_cache_;
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index 51a1d14..69fd1ab 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -76,6 +76,33 @@ TEST_F(DiskInterfaceTest, StatExistingFile) {
   EXPECT_GT(disk_.Stat("file"), 1);
 }
 
+#ifdef _WIN32
+TEST_F(DiskInterfaceTest, StatCache) {
+  disk_.AllowStatCache(true);
+
+  ASSERT_TRUE(Touch("file1"));
+  ASSERT_TRUE(Touch("fiLE2"));
+  ASSERT_TRUE(disk_.MakeDir("subdir"));
+  ASSERT_TRUE(Touch("subdir\\subfile1"));
+  ASSERT_TRUE(Touch("subdir\\SUBFILE2"));
+  ASSERT_TRUE(Touch("subdir\\SUBFILE3"));
+
+  EXPECT_GT(disk_.Stat("FIle1"), 1);
+  EXPECT_GT(disk_.Stat("file1"), 1);
+
+  EXPECT_GT(disk_.Stat("subdir/subfile2"), 1);
+  EXPECT_GT(disk_.Stat("sUbdir\\suBFile1"), 1);
+
+  // Test error cases.
+  disk_.quiet_ = true;
+  string bad_path("cc:\\foo");
+  EXPECT_EQ(-1, disk_.Stat(bad_path));
+  EXPECT_EQ(-1, disk_.Stat(bad_path));
+  EXPECT_EQ(0, disk_.Stat("nosuchfile"));
+  EXPECT_EQ(0, disk_.Stat("nosuchdir/nosuchfile"));
+}
+#endif
+
 TEST_F(DiskInterfaceTest, ReadFile) {
   string err;
   EXPECT_EQ("", disk_.ReadFile("foobar", &err));
diff --git a/src/ninja.cc b/src/ninja.cc
index eedfec0..e555df4 100644
--- a/src/ninja.cc
+++ b/src/ninja.cc
@@ -889,7 +889,7 @@ int NinjaMain::RunBuild(int argc, char** argv) {
     return 1;
   }
 
-  disk_interface_.AllowCache(g_experimental_win_statcache);
+  disk_interface_.AllowStatCache(g_experimental_win_statcache);
 
   Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_);
   for (size_t i = 0; i < targets.size(); ++i) {
@@ -905,7 +905,7 @@ int NinjaMain::RunBuild(int argc, char** argv) {
   }
 
   // Make sure restat rules do not see stale timestamps.
-  disk_interface_.AllowCache(false);
+  disk_interface_.AllowStatCache(false);
 
   if (builder.AlreadyUpToDate()) {
     printf("ninja: no work to do.\n");
-- 
cgit v0.12


From 13cdf9b1812397a969288c755c99926b4011a955 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sun, 15 Jun 2014 15:16:50 -0700
Subject: make bool exist everywhere, for simpler calling code

---
 src/debug_flags.h    | 2 --
 src/disk_interface.h | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/debug_flags.h b/src/debug_flags.h
index ce3c292..4ffef75 100644
--- a/src/debug_flags.h
+++ b/src/debug_flags.h
@@ -26,8 +26,6 @@ extern bool g_explaining;
 
 extern bool g_keep_rsp;
 
-#ifdef _WIN32
 extern bool g_experimental_win_statcache;
-#endif  // _WIN32
 
 #endif // NINJA_EXPLAIN_H_
diff --git a/src/disk_interface.h b/src/disk_interface.h
index a8a3023..0f6dfd3 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -71,7 +71,7 @@ struct RealDiskInterface : public DiskInterface {
   /// Whether to print on errors.  Used to make a test quieter.
   bool quiet_;
 
-  /// Whether stat information can be cached.
+  /// Whether stat information can be cached.  Only has an effect on Windows.
   void AllowStatCache(bool allow);
 
  private:
-- 
cgit v0.12


From b503fdc5367de37c8db42442de511c82a2d35399 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sun, 15 Jun 2014 15:26:37 -0700
Subject: add missing _WIN32 checks

---
 src/disk_interface.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index c4531e6..6437761 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -55,6 +55,7 @@ int MakeDir(const string& path) {
 #endif
 }
 
+#ifdef _WIN32
 TimeStamp TimeStampFromFileTime(const FILETIME& filetime) {
   // FILETIME is in 100-nanosecond increments since the Windows epoch.
   // We don't much care about epoch correctness but we do want the
@@ -109,6 +110,7 @@ bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
   FindClose(hFind);
   return true;
 }
+#endif  // _WIN32
 
 }  // namespace
 
-- 
cgit v0.12


From f6f86d38c09e7caa2e386ce132a63960480ea48b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Sun, 15 Jun 2014 16:00:13 -0700
Subject: s/hFind/find_handle/

---
 src/disk_interface.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 6437761..50a2d97 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -86,10 +86,10 @@ bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
                        bool quiet) {
   // FindExInfoBasic is 30% faster than FindExInfoStandard.
   WIN32_FIND_DATAA ffd;
-  HANDLE hFind = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic, &ffd,
-                                  FindExSearchNameMatch, NULL, 0);
+  HANDLE find_handle = FindFirstFileExA((dir + "\\*").c_str(), FindExInfoBasic,
+                                        &ffd, FindExSearchNameMatch, NULL, 0);
 
-  if (hFind == INVALID_HANDLE_VALUE) {
+  if (find_handle == INVALID_HANDLE_VALUE) {
     DWORD err = GetLastError();
     if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND)
       return true;
@@ -106,8 +106,8 @@ bool StatAllFilesInDir(const string& dir, map<string, TimeStamp>* stamps,
     transform(lowername.begin(), lowername.end(), lowername.begin(), ::tolower);
     stamps->insert(make_pair(lowername,
                              TimeStampFromFileTime(ffd.ftLastWriteTime)));
-  } while (FindNextFileA(hFind, &ffd));
-  FindClose(hFind);
+  } while (FindNextFileA(find_handle, &ffd));
+  FindClose(find_handle);
   return true;
 }
 #endif  // _WIN32
-- 
cgit v0.12