Subninja chdir: parse and load declarations

This builds on the previous CL to implement the parsing of
a subninja chdir, then loading the .ninja file in the
chdir. Note the order of the steps:

1. The subninja chdir is fully parsed in the parent
2. The subninja file is loaded into memory
3. The chdir() is performed
4. LoadManifestTree() is called
5. The current dir is restored like it was

This CL does not implement all the changes to be able to set
parent edges pointing at the subninja.

Change-Id: I7d75f72b252b8f60a6d2d7cec763a879e2c705be

9 files changed, 306 insertions, 48 deletions

diff --git a/src/disk_interface.cc b/src/disk_interface.cc
index 47a02cc..87b74e5 100644
--- a/src/disk_interface.cc
+++ b/src/disk_interface.cc
@@ -25,7 +25,9 @@
 #ifdef _WIN32
 #include <sstream>
 #include <windows.h>
-#include <direct.h>  // _mkdir
+#include <direct.h>  // _mkdir, chdir, getcwd
+#else
+#include <unistd.h>  // chdir, getcwd
 #endif
 
 #include "metrics.h"
@@ -350,3 +352,26 @@ void RealDiskInterface::AllowStatCache(bool allow) {
     cache_.clear();
 #endif
 }
+
+bool RealDiskInterface::Getcwd(std::string* out_path, std::string* err) {
+  vector<char> cwd;
+  do {
+    cwd.resize(cwd.size() + 1024);
+    errno = 0;
+  } while (!::getcwd(&cwd[0], cwd.size()) && errno == ERANGE);
+  if (errno == 0) {
+    out_path->assign(&cwd[0]);
+    err->clear();
+    return true;
+  }
+  *err = "getcwd failed: ";
+  *err += strerror(errno);
+  return false;
+}
+
+bool RealDiskInterface::Chdir(const std::string dir, std::string* err) {
+  if (chdir(dir.c_str()) == 0)
+    return true;
+  *err = "chdir(" + dir + ") failed: " + strerror(errno);
+  return false;
+}
diff --git a/src/disk_interface.h b/src/disk_interface.h
index 66083c6..044ec2b 100644
--- a/src/disk_interface.h
+++ b/src/disk_interface.h
@@ -96,6 +96,13 @@ struct FileReader {
   virtual Status LoadFile(const std::string& path,
                           std::unique_ptr<LoadedFile>* result,
                           std::string* err) = 0;
+
+  // Get the current working directory.  On success, return true.
+  // TODO: use fork() instead of Getcwd().
+  virtual bool Getcwd(std::string* out_path, std::string* err) = 0;
+
+  // Change the current working directory.  On success, return true.
+  virtual bool Chdir(const std::string dir, std::string* err) = 0;
 };
 
 /// Interface for accessing the disk.
@@ -152,6 +159,8 @@ struct RealDiskInterface : public DiskInterface {
   virtual Status LoadFile(const std::string& path,
                           std::unique_ptr<LoadedFile>* result,
                           std::string* err);
+  virtual bool Getcwd(std::string* out_path, std::string* err);
+  virtual bool Chdir(const std::string dir, std::string* err);
   virtual int RemoveFile(const string& path);
 
   /// Whether stat information can be cached.  Only has an effect on Windows.
diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc
index fe8ab73..c0ec67a 100644
--- a/src/disk_interface_test.cc
+++ b/src/disk_interface_test.cc
@@ -237,6 +237,14 @@ struct StatTest : public StateTestWithBuiltinRules,
     assert(false);
     return NotFound;
   }
+  virtual bool Getcwd(std::string* out_path, std::string* err) {
+    assert(false);
+    return false;
+  }
+  virtual bool Chdir(const std::string dir, std::string* err) {
+    assert(false);
+    return false;
+  }
   virtual int RemoveFile(const string& path) {
     assert(false);
     return 0;
diff --git a/src/manifest_chunk_parser.cc b/src/manifest_chunk_parser.cc
index ff36752..617b63e 100644
--- a/src/manifest_chunk_parser.cc
+++ b/src/manifest_chunk_parser.cc
@@ -110,6 +110,25 @@ bool ChunkParser::ParseFileInclude(bool new_scope) {
   include->diag_pos_ = lexer_.GetLastTokenOffset();
   if (!ExpectToken(Lexer::NEWLINE))
     return false;
+  // If new_scope == false, it would be possible to just ignore the next token.
+  // If a Lexer::INDENT somehow was the next token, it would fail with
+  // 'ninja: build.ninja:NNN: unexpected indent'. This might be a slightly more
+  // helpful message.
+  if (lexer_.PeekToken(Lexer::INDENT)) {
+    if (!new_scope)
+      return LexerError("indent after 'include' line is invalid.");
+    if (!lexer_.PeekToken(Lexer::CHDIR))
+      return LexerError("only 'chdir =' is allowed after 'subninja' line.");
+    if (!ExpectToken(Lexer::EQUALS))
+      return LexerError("only 'chdir =' is allowed after 'subninja' line.");
+    if (!lexer_.ReadPath(&include->chdir_, &err))
+      return OutError(err);
+
+    // The trailing '/' is added in manifest_parser.cc.
+    include->chdir_plus_slash_ = include->chdir_.str_.AsString();
+    OutItem(include);
+    return true;
+  }
   OutItem(include);
   return true;
 }
diff --git a/src/manifest_chunk_parser.h b/src/manifest_chunk_parser.h
index 665a805..bda2a2d 100644
--- a/src/manifest_chunk_parser.h
+++ b/src/manifest_chunk_parser.h
@@ -40,6 +40,8 @@ struct RequiredVersion {
 
 struct Include {
   LexedPath path_;
+  LexedPath chdir_;
+  std::string chdir_plus_slash_;
   bool new_scope_ = false;
   size_t diag_pos_ = 0;
 };
diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc
index ad61e20..7b5febc 100644
--- a/src/manifest_parser.cc
+++ b/src/manifest_parser.cc
@@ -95,6 +95,10 @@ struct ManifestFileSet {
     loaded_files_.clear();
   }
 
+  // TODO: use fork() so Getcwd() is no longer needed.
+  bool Getcwd(std::string* out_path, std::string* err);
+  bool Chdir(const std::string dir, std::string* err);
+
 private:
   FileReader *file_reader_ = nullptr;
 
@@ -104,6 +108,14 @@ private:
   std::vector<std::unique_ptr<LoadedFile>> loaded_files_;
 };
 
+bool ManifestFileSet::Getcwd(std::string* out_path, std::string* err) {
+  return file_reader_->Getcwd(out_path, err);
+}
+
+bool ManifestFileSet::Chdir(const std::string dir, std::string* err) {
+  return file_reader_->Chdir(dir, err);
+}
+
 bool ManifestFileSet::LoadFile(const std::string& filename,
                                const LoadedFile** result,
                                std::string* err) {
@@ -123,9 +135,12 @@ struct DfsParser {
 
 private:
   void HandleRequiredVersion(const RequiredVersion& item, Scope* scope);
-  bool HandleInclude(const Include& item, const LoadedFile& file, Scope* scope,
+  bool HandleInclude(Include& item, const LoadedFile& file, Scope* scope,
                      const LoadedFile** child_file, Scope** child_scope,
                      std::string* err);
+  bool LoadIncludeOrSubninja(Include& include, const LoadedFile& file,
+                             Scope* scope, std::vector<Clump*>* out_clumps,
+                             std::string* err);
   bool HandlePool(Pool* pool, const LoadedFile& file, std::string* err);
   bool HandleClump(Clump* clump, const LoadedFile& file, Scope* scope,
                    std::string* err);
@@ -150,7 +165,7 @@ void DfsParser::HandleRequiredVersion(const RequiredVersion& item,
   CheckNinjaVersion(version);
 }
 
-bool DfsParser::HandleInclude(const Include& include, const LoadedFile& file,
+bool DfsParser::HandleInclude(Include& include, const LoadedFile& file,
                               Scope* scope, const LoadedFile** child_file,
                               Scope** child_scope, std::string* err) {
   std::string path;
@@ -158,7 +173,20 @@ bool DfsParser::HandleInclude(const Include& include, const LoadedFile& file,
   if (!file_set_->LoadFile(path, child_file, err)) {
     return DecorateError(file, include.diag_pos_, std::string(*err), err);
   }
-  if (include.new_scope_) {
+  if (!include.chdir_plus_slash_.empty()) {
+    // Evaluate chdir expression and add '/' so the following is always valid:
+    // chdir_plus_slash_ + node.path() == node.globalPath()
+    include.chdir_plus_slash_.clear();
+    EvaluatePathInScope(&include.chdir_plus_slash_, include.chdir_,
+                        scope->GetCurrentEndOfScope());
+    include.chdir_plus_slash_ += '/';
+    include.chdir_.str_ = StringPiece(include.chdir_plus_slash_);
+
+    // Create scope so that subninja chdir variable lookups cannot see parent_
+    *child_scope = new Scope(nullptr);
+
+    Warning("subninja chdir: depending on chdir rules not ready yet.");
+  } else if (include.new_scope_) {
     *child_scope = new Scope(scope->GetCurrentEndOfScope());
   } else {
     *child_scope = scope;
@@ -166,6 +194,46 @@ bool DfsParser::HandleInclude(const Include& include, const LoadedFile& file,
   return true;
 }
 
+bool DfsParser::LoadIncludeOrSubninja(Include& include, const LoadedFile& file,
+                                      Scope* scope,
+                                      std::vector<Clump*>* out_clumps,
+                                      std::string* err) {
+  const LoadedFile* child_file = nullptr;
+  Scope* child_scope = nullptr;
+  std::string prev_cwd;
+
+  if (!HandleInclude(include, file, scope, &child_file, &child_scope, err))
+    return false;
+
+  if (!include.chdir_plus_slash_.empty()) {
+    // The subninja chdir must be treated the same as if the ninja
+    // invocation were done solely inside the chdir. Save the current
+    // working dir and chdir into the subninja.
+    if (!file_set_->Getcwd(&prev_cwd, err)) {
+      *err = "subninja chdir \"" + include.chdir_plus_slash_ + "\": " + *err;
+      return false;
+    }
+    if (!file_set_->Chdir(include.chdir_plus_slash_, err)) {
+      *err = "subninja chdir \"" + include.chdir_plus_slash_ + "\": " + *err;
+      return false;
+    }
+  }
+
+  if (!LoadManifestTree(*child_file, child_scope, out_clumps, err))
+    return false;
+
+  if (!include.chdir_plus_slash_.empty()) {
+    // Restore the directory used by the parent of the subninja chdir.
+    // TODO: fork() could be used, though fork() and pthreads do not mix.
+    // but that would eliminate the need to restore the directory afterward.
+    if (!file_set_->Chdir(prev_cwd, err)) {
+      *err = "subninja chdir \"" + include.chdir_plus_slash_ + "\": restore " + prev_cwd + ": " + *err;
+      return false;
+    }
+  }
+  return true;
+}
+
 bool DfsParser::HandlePool(Pool* pool, const LoadedFile& file,
                            std::string* err) {
   std::string depth_string;
@@ -220,7 +288,8 @@ bool DfsParser::LoadManifestTree(const LoadedFile& file, Scope* scope,
 
   // With the chunks parsed, do a depth-first parse of the ninja manifest using
   // the results of the parallel parse.
-  for (const auto& item : items) {
+  for (auto& item_nonconst : items) {
+    const auto& item = item_nonconst;
     switch (item.kind) {
     case ParserItem::kError:
       *err = item.u.error->msg_;
@@ -230,16 +299,11 @@ bool DfsParser::LoadManifestTree(const LoadedFile& file, Scope* scope,
       HandleRequiredVersion(*item.u.required_version, scope);
       break;
 
-    case ParserItem::kInclude: {
-      const Include& include = *item.u.include;
-      const LoadedFile* child_file = nullptr;
-      Scope* child_scope = nullptr;
-      if (!HandleInclude(include, file, scope, &child_file, &child_scope, err))
-        return false;
-      if (!LoadManifestTree(*child_file, child_scope, out_clumps, err))
+    case ParserItem::kInclude:
+      if (!LoadIncludeOrSubninja(*item_nonconst.u.include, file, scope,
+                                 out_clumps, err))
         return false;
       break;
-    }
 
     case ParserItem::kClump:
       if (!HandleClump(item.u.clump, file, scope, err))
diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc
index 7a042d2..0f772df 100644
--- a/src/manifest_parser_test.cc
+++ b/src/manifest_parser_test.cc
@@ -938,6 +938,34 @@ TEST_F(ParserTest, SubNinja) {
   EXPECT_EQ("varref outer", state.edges_[2]->EvaluateCommand());
 }
 
+TEST_F(ParserTest, SubNinjaChdir) {
+  EXPECT_TRUE(fs_.MakeDir("a"));
+  fs_.Create("a/foo.ninja",
+    "var = inner\n"
+    "rule innerrule\n"
+    "  command = foo $var\n"
+    "build $builddir/inner: innerrule\n");
+
+  ASSERT_NO_FATAL_FAILURE(AssertParse(
+"builddir = a/\n"
+"rule varref\n"
+"  command = varref $var\n"
+"var = outer\n"
+"build $builddir/outer: varref\n"
+"subninja a/foo.ninja\n"
+"  chdir = a\n"
+"build $builddir/outer2: varref\n"));
+  ASSERT_EQ(1u, fs_.files_read_.size());
+
+  EXPECT_EQ("a/foo.ninja", fs_.files_read_[0]);
+  EXPECT_TRUE(state.LookupNode("a/outer"));
+  EXPECT_TRUE(state.LookupNode("a/outer2"));
+  // Verify builddir setting is *not* inherited.
+  EXPECT_FALSE(state.LookupNode("a/inner"));
+  EXPECT_FALSE(state.LookupNode("a/" "/inner"));
+  EXPECT_TRUE(state.LookupNode("/inner"));  // $builddir expands to ""
+}
+
 TEST_F(ParserTest, MissingSubNinja) {
   ManifestParser parser(&state, &fs_);
   string err;
@@ -983,15 +1011,48 @@ TEST_F(ParserTest, Include) {
   EXPECT_EQ("inner", state.root_scope_.LookupVariable("var"));
 }
 
-TEST_F(ParserTest, BrokenInclude) {
+TEST_F(ParserTest, IncludeErrors) {
   fs_.Create("include.ninja", "build\n");
-  ManifestParser parser(&state, &fs_);
-  string err;
-  EXPECT_FALSE(parser.ParseTest("include include.ninja\n", &err));
-  EXPECT_EQ("include.ninja:1: expected path\n"
-            "build\n"
-            "     ^ near here"
-            , err);
+  {
+    ManifestParser parser(&state, &fs_);
+    string err;
+    EXPECT_FALSE(parser.ParseTest("include include.ninja\n", &err));
+    EXPECT_EQ("include.ninja:1: expected path\n"
+              "build\n"
+              "     ^ near here"
+              , err);
+  }
+  {
+    ManifestParser parser(&state, &fs_);
+    string err;
+    EXPECT_FALSE(parser.ParseTest(
+        "include include.ninja\n"
+        "  chdir = somedir\n", &err));
+    EXPECT_EQ("input:2: indent after 'include' line is invalid.\n"
+              , err);
+  }
+  {
+    ManifestParser parser(&state, &fs_);
+    string err;
+    EXPECT_FALSE(parser.ParseTest(
+        "subninja include.ninja\n"
+        "  somedir = somedir\n", &err));
+    EXPECT_EQ("input:2: only 'chdir =' is allowed after 'subninja' line.\n"
+              "  somedir = somedir\n"
+              "  ^ near here"
+              , err);
+  }
+  {
+    ManifestParser parser(&state, &fs_);
+    string err;
+    EXPECT_FALSE(parser.ParseTest(
+        "subninja include.ninja\n"
+        "  chdir dir\n", &err));
+    EXPECT_EQ("input:2: expected '=', got identifier\n"
+              "  chdir dir\n"
+              "        ^ near here"
+              , err);
+  }
 }
 
 TEST_F(ParserTest, Implicit) {
diff --git a/src/test.cc b/src/test.cc
index 7cacae4..19e86c7 100644
--- a/src/test.cc
+++ b/src/test.cc
@@ -170,26 +170,29 @@ void VerifyGraph(const State& state) {
 
 void VirtualFileSystem::Create(const string& path,
                                const string& contents) {
-  files_[path].mtime = now_;
-  files_[path].contents = contents;
-  files_created_.insert(path);
+  string fullpath = cwd_ + path;
+  files_[fullpath].mtime = now_;
+  files_[fullpath].contents = contents;
+  files_created_.insert(fullpath);
 }
 
 void VirtualFileSystem::CreateSymlink(const string& path,
                                       const string& dest) {
-  files_[path].mtime = now_;
-  files_[path].contents = dest;
-  files_[path].is_symlink = true;
-  files_created_.insert(path);
+  string fullpath = cwd_ + path;
+  files_[fullpath].mtime = now_;
+  files_[fullpath].contents = dest;
+  files_[fullpath].is_symlink = true;
+  files_created_.insert(fullpath);
 }
 
 TimeStamp VirtualFileSystem::Stat(const string& path, string* err) const {
-  DirMap::const_iterator d = dirs_.find(path);
+  string fullpath = cwd_ + path;
+  DirMap::const_iterator d = dirs_.find(fullpath);
   if (d != dirs_.end()) {
     *err = d->second.stat_error;
     return d->second.mtime;
   }
-  FileMap::const_iterator i = files_.find(path);
+  FileMap::const_iterator i = files_.find(fullpath);
   if (i != files_.end()) {
     if (i->second.is_symlink) {
       return Stat(i->second.contents, err);
@@ -201,14 +204,15 @@ TimeStamp VirtualFileSystem::Stat(const string& path, string* err) const {
 }
 
 TimeStamp VirtualFileSystem::LStat(const string& path, bool* is_dir, bool* is_symlink, string* err) const {
-  DirMap::const_iterator d = dirs_.find(path);
+  string fullpath = cwd_ + path;
+  DirMap::const_iterator d = dirs_.find(fullpath);
   if (d != dirs_.end()) {
     if (is_dir != nullptr)
       *is_dir = true;
     *err = d->second.stat_error;
     return d->second.mtime;
   }
-  FileMap::const_iterator i = files_.find(path);
+  FileMap::const_iterator i = files_.find(fullpath);
   if (i != files_.end()) {
     if (is_dir != nullptr)
       *is_dir = false;
@@ -225,13 +229,14 @@ bool VirtualFileSystem::IsStatThreadSafe() const {
 }
 
 bool VirtualFileSystem::WriteFile(const string& path, const string& contents) {
-  if (files_.find(path) == files_.end()) {
-    if (dirs_.find(path) != dirs_.end())
+  string fullpath = cwd_ + path;
+  if (files_.find(fullpath) == files_.end()) {
+    if (dirs_.find(fullpath) != dirs_.end())
       return false;
 
-    string::size_type slash_pos = path.find_last_of("/");
+    string::size_type slash_pos = fullpath.find_last_of("/");
     if (slash_pos != string::npos) {
-      DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+      DirMap::iterator d = dirs_.find(fullpath.substr(0, slash_pos));
       if (d != dirs_.end()) {
         d->second.mtime = now_;
       } else {
@@ -245,14 +250,15 @@ bool VirtualFileSystem::WriteFile(const string& path, const string& contents) {
 }
 
 bool VirtualFileSystem::MakeDir(const string& path) {
-  if (dirs_.find(path) != dirs_.end())
+  string fullpath = cwd_ + path;
+  if (dirs_.find(fullpath) != dirs_.end())
     return true;
-  if (files_.find(path) != files_.end())
+  if (files_.find(fullpath) != files_.end())
     return false;
 
-  string::size_type slash_pos = path.find_last_of("/");
+  string::size_type slash_pos = fullpath.find_last_of("/");
   if (slash_pos != string::npos) {
-    DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+    DirMap::iterator d = dirs_.find(fullpath.substr(0, slash_pos));
     if (d != dirs_.end()) {
       d->second.mtime = now_;
     } else {
@@ -260,8 +266,8 @@ bool VirtualFileSystem::MakeDir(const string& path) {
     }
   }
 
-  dirs_[path].mtime = now_;
-  directories_made_.push_back(path);
+  dirs_[fullpath].mtime = now_;
+  directories_made_.push_back(fullpath);
   return true;  // success
 }
 
@@ -280,8 +286,9 @@ FileReader::Status VirtualFileSystem::ReadFile(const string& path,
 FileReader::Status VirtualFileSystem::LoadFile(const std::string& path,
                                                std::unique_ptr<LoadedFile>* result,
                                                std::string* err) {
-  files_read_.push_back(path);
-  FileMap::iterator i = files_.find(path);
+  string fullpath = cwd_ + path;
+  files_read_.push_back(fullpath);
+  FileMap::iterator i = files_.find(fullpath);
   if (i != files_.end()) {
     if (i->second.is_symlink) {
       return LoadFile(i->second.contents, result, err);
@@ -295,14 +302,71 @@ FileReader::Status VirtualFileSystem::LoadFile(const std::string& path,
   return NotFound;
 }
 
+bool VirtualFileSystem::Getcwd(std::string* out_path, std::string* err) {
+  // Empty cwd_ means the current dir is '/'
+  if (cwd_.empty()) {
+    out_path->assign("/");
+  } else {
+    out_path->assign(cwd_);
+    // Strip off '/' if present.
+    if (cwd_.size() > 0 && cwd_.at(cwd_.size() - 1) == '/') {
+      out_path->erase(out_path->end() - 1);
+    }
+  }
+  err->clear();
+  return true;
+}
+
+bool VirtualFileSystem::Chdir(const std::string dir, std::string* err) {
+  // VirtualFileSystem::Chdir does not support ".." or "." relative paths.
+  // However, simple relative paths are ok. And absolute paths are ok.
+
+  string dest;
+  if (dir.empty()) {
+    err->assign("VirtualFileSystem::Chdir does not accept the empty string");
+    return false;
+  } else if (dir == "/") {
+    cwd_.clear();
+    err->clear();
+    return true;
+  } else if (dir.at(0) == '/') {
+    // Treat path as absolute.
+    dest.insert(dest.begin(), dir.begin() + 1, dir.end());
+  } else {
+    // Treat path as relative.
+    dest = cwd_ + dir;
+  }
+  if (dest.find(".") != string::npos) {
+    err->assign("VirtualFileSystem::Chdir does not accept . or ..");
+    return false;
+  }
+  if (dest.size() > 1 && dest.at(dest.size() - 1) == '/') {
+    dest.erase(dest.size() - 1);
+  }
+
+  // Look up dest in directories_made_.
+  vector<string>::iterator i = directories_made_.begin();
+  for (; i != directories_made_.end(); i++) {
+    if (*i == dest) {
+      cwd_ = dest;
+      cwd_ += '/';
+      err->clear();
+      return true;
+    }
+  }
+  *err = strerror(ENOENT);
+  return false;
+}
+
 int VirtualFileSystem::RemoveFile(const string& path) {
-  if (dirs_.find(path) != dirs_.end())
+  string fullpath = cwd_ + path;
+  if (dirs_.find(fullpath) != dirs_.end())
     return -1;
-  FileMap::iterator i = files_.find(path);
+  FileMap::iterator i = files_.find(fullpath);
   if (i != files_.end()) {
-    string::size_type slash_pos = path.find_last_of("/");
+    string::size_type slash_pos = fullpath.find_last_of("/");
     if (slash_pos != string::npos) {
-      DirMap::iterator d = dirs_.find(path.substr(0, slash_pos));
+      DirMap::iterator d = dirs_.find(fullpath.substr(0, slash_pos));
       if (d != dirs_.end()) {
         d->second.mtime = now_;
       }
diff --git a/src/test.h b/src/test.h
index 4adc3c9..a5fbc18 100644
--- a/src/test.h
+++ b/src/test.h
@@ -156,6 +156,8 @@ struct VirtualFileSystem : public DiskInterface {
   virtual Status LoadFile(const std::string& path,
                           std::unique_ptr<LoadedFile>* result,
                           std::string* err);
+  virtual bool Getcwd(std::string* out_path, std::string* err);
+  virtual bool Chdir(const std::string dir, std::string* err);
   virtual int RemoveFile(const string& path);
 
   /// An entry for a single in-memory file.
@@ -181,6 +183,10 @@ struct VirtualFileSystem : public DiskInterface {
 
   /// A simple fake timestamp for file operations.
   int now_;
+
+  // Current directory for file operations, should end in '/' if not empty so
+  // the following is always valid: fullpath = cwd_ + path.
+  string cwd_;
 };
 
 struct ScopedTempDir {