// Copyright 2015 Google Inc. All rights reserved // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build ignore #include "ninja.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "command.h" #include "dep.h" #include "eval.h" #include "file_cache.h" #include "fileutil.h" #include "find.h" #include "flags.h" #include "func.h" #include "io.h" #include "log.h" #include "stats.h" #include "stringprintf.h" #include "strutil.h" #include "timeutil.h" #include "var.h" #include "version.h" static size_t FindCommandLineFlag(std::string_view cmd, std::string_view name) { const size_t found = cmd.find(name); if (found == std::string::npos || found == 0) return std::string::npos; return found; } static std::string_view FindCommandLineFlagWithArg(std::string_view cmd, std::string_view name) { size_t index = FindCommandLineFlag(cmd, name); if (index == std::string::npos) return std::string_view(); std::string_view val = TrimLeftSpace(cmd.substr(index + name.size())); index = val.find(name); while (index != std::string::npos) { val = TrimLeftSpace(val.substr(index + name.size())); index = val.find(name); } index = val.find_first_of(" \t"); return val.substr(0, index); } static bool StripPrefix(std::string_view p, std::string_view* s) { if (!HasPrefix(*s, p)) return false; *s = s->substr(p.size()); return true; } size_t GetGomaccPosForAndroidCompileCommand(std::string_view cmdline) { size_t index = cmdline.find(' '); if (index == std::string::npos) return std::string::npos; std::string_view cmd = cmdline.substr(0, index); if (HasSuffix(cmd, "ccache")) { index++; size_t pos = GetGomaccPosForAndroidCompileCommand(cmdline.substr(index)); return pos == std::string::npos ? std::string::npos : pos + index; } if (!StripPrefix("prebuilts/", &cmd)) return std::string::npos; if (!StripPrefix("gcc/", &cmd) && !StripPrefix("clang/", &cmd)) return std::string::npos; if (!HasSuffix(cmd, "gcc") && !HasSuffix(cmd, "g++") && !HasSuffix(cmd, "clang") && !HasSuffix(cmd, "clang++")) { return std::string::npos; } std::string_view rest = cmdline.substr(index); return rest.find(" -c ") != std::string::npos ? 0 : std::string::npos; } static bool GetDepfileFromCommandImpl(std::string_view cmd, std::string* out) { if ((FindCommandLineFlag(cmd, " -MD") == std::string::npos && FindCommandLineFlag(cmd, " -MMD") == std::string::npos) || FindCommandLineFlag(cmd, " -c") == std::string::npos) { return false; } std::string_view mf = FindCommandLineFlagWithArg(cmd, " -MF"); if (!mf.empty()) { out->append(mf); return true; } std::string_view o = FindCommandLineFlagWithArg(cmd, " -o"); if (o.empty()) { ERROR("Cannot find the depfile in %s", std::string(cmd).c_str()); return false; } out->append(StripExt(o)); *out += ".d"; return true; } bool GetDepfileFromCommand(std::string* cmd, std::string* out) { CHECK(!cmd->empty()); if (!GetDepfileFromCommandImpl(*cmd, out)) return false; // A hack for Android - llvm-rs-cc seems not to emit a dep file. if (cmd->find("bin/llvm-rs-cc ") != std::string::npos) { return false; } // TODO: A hack for Makefiles generated by automake. // A hack for Android to get .P files instead of .d. std::string p{StripExt(*out)}; p += ".P"; if (cmd->find(p) != std::string::npos) { const std::string rm_f = "; rm -f " + *out; const size_t found = cmd->find(rm_f); if (found == std::string::npos) { ERROR("Cannot find removal of .d file: %s", cmd->c_str()); } cmd->erase(found, rm_f.size()); return true; } // A hack for Android. For .s files, GCC does not use C // preprocessor, so it ignores -MF flag. std::string as = "/"; as.append(StripExt(Basename(*out))); as += ".s"; if (cmd->find(as) != std::string::npos) { return false; } *cmd += "&& cp "; *cmd += *out; *cmd += ' '; *cmd += *out; *cmd += ".tmp "; *out += ".tmp"; return true; } struct NinjaNode { const DepNode* node; std::vector commands; int rule_id; }; class NinjaGenerator { public: NinjaGenerator(Evaluator* ev, double start_time) : ce_(ev), ev_(ev), rule_id_(0), shell_(EscapeNinja(ev->GetShell())), shell_flags_(EscapeNinja(ev->GetShellFlag())), kati_binary_(GetExecutablePath()), start_time_(start_time), default_target_(NULL) { ev_->set_avoid_io(true); const std::string use_goma_str = ev->EvalVar(Intern("USE_GOMA")); use_goma_ = !(use_goma_str.empty() || use_goma_str == "false"); if (g_flags.goma_dir) gomacc_ = StringPrintf("%s/gomacc ", g_flags.goma_dir); } ~NinjaGenerator() { ev_->set_avoid_io(false); } void Generate(const std::vector& nodes, const std::string& orig_args) { unlink(GetNinjaStampFilename().c_str()); PopulateNinjaNodes(nodes); GenerateNinja(); GenerateShell(); GenerateStamp(orig_args); } static std::string GetStampTempFilename() { return GetFilename(".kati_stamp%s.tmp"); } static std::string GetFilename(const char* fmt) { std::string r = g_flags.ninja_dir ? g_flags.ninja_dir : "."; r += '/'; r += StringPrintf(fmt, g_flags.ninja_suffix ? g_flags.ninja_suffix : ""); return r; } private: void PopulateNinjaNodes(const std::vector& nodes) { ScopedTimeReporter tr("ninja gen (eval)"); for (auto const& [symbol, depnode] : nodes) { PopulateNinjaNode(*depnode); } } void PopulateNinjaNode(const DepNode& node) { if (done_.exists(node.output)) { return; } done_.insert(node.output); ScopedFrame frame( ce_.evaluator()->Enter(FrameType::NINJA, node.output.str(), node.loc)); // A hack to exclude out phony target in Android. If this exists, // "ninja -t clean" tries to remove this directory and fails. if (g_flags.detect_android_echo && node.output.str() == "out") return; // This node is a leaf node if (!node.has_rule && !node.is_phony) { return; } NinjaNode& nn = nodes_.emplace_back(); nn.node = &node; nn.commands = ce_.Eval(node); nn.rule_id = nn.commands.empty() ? -1 : rule_id_++; for (auto const& [symbol, depnode] : node.deps) { PopulateNinjaNode(*depnode); } for (auto const& [symbol, depnode] : node.order_onlys) { PopulateNinjaNode(*depnode); } for (auto const& [symbol, depnode] : node.validations) { PopulateNinjaNode(*depnode); } } std::string_view TranslateCommand(const char* in, std::string* cmd_buf) { const size_t orig_size = cmd_buf->size(); bool prev_backslash = false; // Set space as an initial value so the leading comment will be // stripped out. char prev_char = ' '; char quote = 0; for (; *in; in++) { switch (*in) { case '#': if (quote == 0 && isspace(prev_char)) { while (in[1] && *in != '\n') in++; } else { *cmd_buf += *in; } break; case '\'': case '"': case '`': if (quote) { if (quote == *in) quote = 0; } else if (!prev_backslash) { quote = *in; } *cmd_buf += *in; break; case '$': *cmd_buf += "$$"; break; case '\n': if (prev_backslash) { cmd_buf->resize(cmd_buf->size() - 1); } else { *cmd_buf += ' '; } break; case '\\': *cmd_buf += '\\'; break; default: *cmd_buf += *in; } if (*in == '\\') { prev_backslash = !prev_backslash; } else { prev_backslash = false; } prev_char = *in; } if (prev_backslash) { cmd_buf->resize(cmd_buf->size() - 1); } while (true) { char c = (*cmd_buf)[cmd_buf->size() - 1]; if (!isspace(c) && c != ';') break; cmd_buf->resize(cmd_buf->size() - 1); } return std::string_view(cmd_buf->data() + orig_size, cmd_buf->size() - orig_size); } bool IsOutputMkdir(const char* name, std::string_view cmd) { if (!HasPrefix(cmd, "mkdir -p ")) { return false; } cmd = cmd.substr(9, cmd.size()); if (!cmd.empty() && cmd.back() == '/') { cmd.remove_suffix(1); } std::string_view dir = Dirname(name); if (cmd == dir) { return true; } return false; } bool GetDescriptionFromCommand(std::string_view cmd, std::string* out) { if (!HasPrefix(cmd, "echo ")) { return false; } cmd = cmd.substr(5, cmd.size()); bool prev_backslash = false; char quote = 0; std::string out_buf; // Strip outer quotes, and fail if it is not a single echo command for (std::string_view::iterator in = cmd.begin(); in != cmd.end(); in++) { if (prev_backslash) { prev_backslash = false; out_buf += *in; } else if (*in == '\\') { prev_backslash = true; out_buf += *in; } else if (quote) { if (*in == quote) { quote = 0; } else { out_buf += *in; } } else { switch (*in) { case '\'': case '"': case '`': quote = *in; break; case '<': case '>': case '&': case '|': case ';': return false; default: out_buf += *in; } } } *out = out_buf; return true; } bool GenShellScript(const char* name, const std::vector& commands, std::string* cmd_buf, std::string* description) { bool got_descritpion = false; bool use_gomacc = false; auto command_count = commands.size(); for (const Command& c : commands) { size_t cmd_begin = cmd_buf->size(); if (!cmd_buf->empty()) { *cmd_buf += " && "; } const char* in = c.cmd.c_str(); while (isspace(*in)) in++; bool needs_subshell = (command_count > 1 || c.ignore_error) && !c.force_no_subshell; if (needs_subshell) *cmd_buf += '('; size_t cmd_start = cmd_buf->size(); std::string_view translated = TranslateCommand(in, cmd_buf); if (g_flags.detect_android_echo && !got_descritpion && !c.echo && GetDescriptionFromCommand(translated, description)) { got_descritpion = true; translated = std::string_view(); } else if (IsOutputMkdir(name, translated) && !c.echo && cmd_begin == 0) { translated = std::string_view(); } if (translated.empty()) { cmd_buf->resize(cmd_begin); command_count -= 1; continue; } else if (g_flags.goma_dir) { size_t pos = GetGomaccPosForAndroidCompileCommand(translated); if (pos != std::string::npos) { cmd_buf->insert(cmd_start + pos, gomacc_); use_gomacc = true; } } else if (translated.find("/gomacc") != std::string::npos) { use_gomacc = true; } if (c.ignore_error) { *cmd_buf += " ; true"; } if (needs_subshell) *cmd_buf += " )"; } return (use_goma_ || g_flags.remote_num_jobs || g_flags.goma_dir) && !use_gomacc; } bool GetDepfile(const DepNode* node, std::string* cmd_buf, std::string* depfile) { if (node->depfile_var) { node->depfile_var->Eval(ev_, depfile); return true; } if (!g_flags.detect_depfiles) return false; *cmd_buf += ' '; bool result = GetDepfileFromCommand(cmd_buf, depfile); cmd_buf->resize(cmd_buf->size() - 1); return result; } void EmitDepfile(const NinjaNode& nn, std::string* cmd_buf, std::ostream& out) { const DepNode* node = nn.node; std::string depfile; if (!GetDepfile(node, cmd_buf, &depfile)) return; out << " depfile = " << depfile << "\n"; out << " deps = gcc\n"; } void EmitNode(const NinjaNode& nn, std::ostream& out) { const DepNode* node = nn.node; const std::vector& commands = nn.commands; std::string rule_name = "phony"; bool use_local_pool = false; if (IsSpecialTarget(node->output)) { return; } if (g_flags.enable_debug) { out << "# " << (node->loc.filename ? node->loc.filename : "(null)") << ':' << node->loc.lineno << "\n"; } if (!commands.empty()) { rule_name = StringPrintf("rule%d", nn.rule_id); out << "rule " << rule_name << "\n"; std::string description = "build $out"; std::string cmd_buf; use_local_pool |= GenShellScript(node->output.c_str(), commands, &cmd_buf, &description); out << " description = " << description << "\n"; EmitDepfile(nn, &cmd_buf, out); // It seems Linux is OK with ~130kB and Mac's limit is ~250kB. // TODO: Find this number automatically. if (cmd_buf.size() > 100 * 1000) { out << " rspfile = $out.rsp\n"; out << " rspfile_content = " << cmd_buf << "\n"; out << " command = " << shell_ << " $out.rsp\n"; } else { EscapeShell(&cmd_buf); out << " command = " << shell_ << ' ' << shell_flags_ << " \"" << cmd_buf << "\"\n"; } if (node->is_restat) { out << " restat = 1\n"; } } EmitBuild(nn, rule_name, use_local_pool, out); } std::string EscapeNinja(const std::string& s) const { if (s.find_first_of("$: ") == std::string::npos) return s; std::string r; for (char c : s) { switch (c) { case '$': case ':': case ' ': r += '$'; [[fallthrough]]; default: r += c; } } return r; } std::string EscapeBuildTarget(Symbol s) const { return EscapeNinja(s.str()); } void EmitBuild(const NinjaNode& nn, const std::string& rule_name, bool use_local_pool, std::ostream& out) { const DepNode* node = nn.node; std::string target = EscapeBuildTarget(node->output); out << "build " << target; if (!node->implicit_outputs.empty()) { out << " |"; for (Symbol output : node->implicit_outputs) { out << " " << EscapeBuildTarget(output); } } out << ": " << rule_name; std::vector order_onlys; if (node->is_phony && !g_flags.use_ninja_phony_output) { out << " _kati_always_build_"; } for (auto const& d : node->deps) { out << " " << EscapeBuildTarget(d.first).c_str(); } if (!node->order_onlys.empty()) { out << " ||"; for (auto const& d : node->order_onlys) { out << " " << EscapeBuildTarget(d.first).c_str(); } } if (!node->validations.empty()) { out << " |@"; for (auto const& d : node->validations) { out << " " << EscapeBuildTarget(d.first).c_str(); } } out << "\n"; if (!node->symlink_outputs.empty()) { out << " symlink_outputs ="; for (auto const& s : node->symlink_outputs) { out << " " << EscapeBuildTarget(s); } out << "\n"; } std::string pool; if (node->ninja_pool_var) { node->ninja_pool_var->Eval(ev_, &pool); } if (pool != "") { if (pool != "none") { out << " pool = " << pool << "\n"; } } else if (g_flags.default_pool && rule_name != "phony") { out << " pool = " << g_flags.default_pool << "\n"; } else if (use_local_pool) { out << " pool = local_pool\n"; } if (node->is_phony && g_flags.use_ninja_phony_output) { out << " phony_output = true\n"; } if (node->tags_var) { std::string tags; node->tags_var->Eval(ev_, &tags); if (!tags.empty()) { out << " tags = " << tags << "\n"; } } if (node->is_default_target) { std::unique_lock lock(mu_); default_target_ = node; } } static std::string GetEnvScriptFilename() { return GetFilename("env%s.sh"); } void GenerateNinja() { ScopedTimeReporter tr("ninja gen (emit)"); std::ofstream out(GetNinjaFilename(), std::ios::binary); if (!out) PERROR("fopen(build.ninja) failed"); out << "# Generated by kati " << kGitVersion << "\n\n"; if (!used_envs_.empty()) { out << "# Environment variables used:\n"; for (const auto& [key, value] : used_envs_) { out << "# " << key << "=" << value << '\n'; } out << '\n'; } if (!g_flags.no_ninja_prelude) { if (g_flags.ninja_dir) { out << "builddir = " << g_flags.ninja_dir << "\n\n"; } out << "pool local_pool\n" << " depth = " << g_flags.num_jobs << "\n\n"; if (!g_flags.use_ninja_phony_output) { out << "build _kati_always_build_: phony\n\n"; } } if (!g_flags.generate_empty_ninja) { for (const auto& node : nodes_) { EmitNode(node, out); } std::string default_targets; if (g_flags.targets.empty() || g_flags.gen_all_targets) { CHECK(default_target_); default_targets = EscapeBuildTarget(default_target_->output); } else { for (Symbol s : g_flags.targets) { if (!default_targets.empty()) default_targets += ' '; default_targets += EscapeBuildTarget(s); } } out << "\n" << "default " << default_targets << '\n'; } SymbolSet used_env_vars(Vars::used_env_vars()); // PATH changes $(shell). used_env_vars.insert(Intern("PATH")); for (Symbol e : used_env_vars) { std::string_view val(getenv(e.c_str())); used_envs_.emplace(e.str(), std::string(val)); } } void GenerateShell() { FILE* fp = fopen(GetEnvScriptFilename().c_str(), "wb"); if (fp == NULL) PERROR("fopen(env.sh) failed"); fprintf(fp, "#!/bin/sh\n"); fprintf(fp, "# Generated by kati %s\n", kGitVersion); fprintf(fp, "\n"); for (const auto& [symbol, is_exported] : ev_->exports()) { if (is_exported) { const std::string val = ev_->EvalVar(symbol); fprintf(fp, "export '%s'='%s'\n", symbol.c_str(), val.c_str()); } else { fprintf(fp, "unset '%s'\n", symbol.c_str()); } } fclose(fp); fp = fopen(GetNinjaShellScriptFilename().c_str(), "wb"); if (fp == NULL) PERROR("fopen(ninja.sh) failed"); fprintf(fp, "#!/bin/sh\n"); fprintf(fp, "# Generated by kati %s\n", kGitVersion); fprintf(fp, "\n"); fprintf(fp, ". %s\n", GetEnvScriptFilename().c_str()); fprintf(fp, "exec ninja -f %s ", GetNinjaFilename().c_str()); if (g_flags.remote_num_jobs > 0) { fprintf(fp, "-j%d ", g_flags.remote_num_jobs); } else if (g_flags.goma_dir) { fprintf(fp, "-j500 "); } fprintf(fp, "\"$@\"\n"); fclose(fp); if (chmod(GetNinjaShellScriptFilename().c_str(), 0755) != 0) PERROR("chmod ninja.sh failed"); } void GenerateStamp(const std::string& orig_args) { FILE* fp = fopen(GetStampTempFilename().c_str(), "wb"); CHECK(fp); size_t r = fwrite(&start_time_, sizeof(start_time_), 1, fp); CHECK(r == 1); std::unordered_set makefiles; MakefileCacheManager::Get().GetAllFilenames(&makefiles); DumpInt(fp, makefiles.size() + 1); DumpString(fp, kati_binary_); for (const std::string& makefile : makefiles) { DumpString(fp, makefile); } DumpInt(fp, Evaluator::used_undefined_vars().size()); for (Symbol v : Evaluator::used_undefined_vars()) { DumpString(fp, v.str()); } DumpInt(fp, used_envs_.size()); for (const auto& [key, value] : used_envs_) { DumpString(fp, key); DumpString(fp, value); } auto& globs = GetAllGlobCache(); DumpInt(fp, globs.size()); for (const auto& [key, files] : globs) { DumpString(fp, key); #if 0 std::unordered_set dirs; GetReadDirs(p.first, files, &dirs); DumpInt(fp, dirs.size()); for (const std::string& dir : dirs) { DumpString(fp, dir); } #endif DumpInt(fp, files.size()); for (const std::string& file : files) { DumpString(fp, file); } } const std::vector& crs = GetShellCommandResults(); DumpInt(fp, crs.size()); for (CommandResult* cr : crs) { DumpInt(fp, static_cast(cr->op)); DumpString(fp, cr->shell); DumpString(fp, cr->shellflag); DumpString(fp, cr->cmd); DumpString(fp, cr->result); DumpString(fp, cr->loc.filename); DumpInt(fp, cr->loc.lineno); if (cr->op == CommandOp::FIND) { std::vector missing_dirs; for (std::string_view fd : cr->find->finddirs) { const std::string& d = ConcatDir(cr->find->chdir, fd); if (!Exists(d)) missing_dirs.push_back(d); } DumpInt(fp, missing_dirs.size()); for (const std::string& d : missing_dirs) { DumpString(fp, d); } DumpInt(fp, cr->find->found_files->size()); for (std::string_view s : *cr->find->found_files) { DumpString(fp, ConcatDir(cr->find->chdir, s)); } DumpInt(fp, cr->find->read_dirs->size()); for (std::string_view s : *cr->find->read_dirs) { DumpString(fp, ConcatDir(cr->find->chdir, s)); } } } DumpString(fp, orig_args); fclose(fp); rename(GetStampTempFilename().c_str(), GetNinjaStampFilename().c_str()); } CommandEvaluator ce_; Evaluator* ev_; SymbolSet done_; int rule_id_; bool use_goma_; std::string gomacc_; const std::string shell_; const std::string shell_flags_; std::map used_envs_; const std::string kati_binary_; const double start_time_; std::vector nodes_; std::mutex mu_; const DepNode* default_target_; }; std::string GetNinjaFilename() { return NinjaGenerator::GetFilename("build%s.ninja"); } std::string GetNinjaShellScriptFilename() { return NinjaGenerator::GetFilename("ninja%s.sh"); } std::string GetNinjaStampFilename() { return NinjaGenerator::GetFilename(".kati_stamp%s"); } void GenerateNinja(const std::vector& nodes, Evaluator* ev, const std::string& orig_args, double start_time) { NinjaGenerator ng(ev, start_time); ng.Generate(nodes, orig_args); }