diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:24:26 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:24:26 +0000 |
commit | cd1629245da56b1fa6e0bc24f34658607c5d1efa (patch) | |
tree | 3db9be87e973a94975d18d459a2428a086cad898 | |
parent | cf4155fa75a56d6425caa897f785fb3839bdb482 (diff) | |
parent | 8cefae833949cddadf7068f8bcab5511329ede38 (diff) | |
download | treble-android14-mainline-uwb-release.tar.gz |
Snap for 10453563 from 8cefae833949cddadf7068f8bcab5511329ede38 to mainline-uwb-releaseaml_uwb_341513070aml_uwb_341511050aml_uwb_341310300aml_uwb_341310030aml_uwb_341111010aml_uwb_341011000android14-mainline-uwb-release
Change-Id: Ie9750bddbd6ef9ed90b0f00c885c13892bf1e2a3
35 files changed, 2965 insertions, 171 deletions
@@ -1,31 +1,3 @@ -// -// Copyright (C) 2021 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package { - default_applicable_licenses: ["tools_treble_license"], -} - -// Added automatically by a large-scale-change -// See: http://go/android-license-faq -license { - name: "tools_treble_license", - visibility: [":__subpackages__"], - license_kinds: [ - "SPDX-license-identifier-Apache-2.0", - ], - license_text: [ - "LICENSE", - ], + default_applicable_licenses: ["Android-Apache-2.0"], } @@ -1,5 +1,5 @@ -diegowilson@google.com -jjdemartino@google.com -lavers@google.com kiyoungkim@google.com -danielnorman@google.com +deyaoren@google.com +haamed@google.com +jgalmes@google.com +rseymour@google.com
\ No newline at end of file diff --git a/build/Android.bp b/build/Android.bp index 7d1731e..32c5bf5 100644 --- a/build/Android.bp +++ b/build/Android.bp @@ -1,37 +1,10 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package { - // See: http://go/android-license-faq - // A large-scale-change added 'default_applicable_licenses' to import - // all of the 'license_kinds' from "tools_treble_license" - // to get the below license kinds: - // SPDX-license-identifier-Apache-2.0 - default_applicable_licenses: ["tools_treble_license"], + default_applicable_licenses: ["Android-Apache-2.0"], } python_defaults { name: "treble_build_default", pkg_path: "treble/build", - version: { - py2: { - enabled: false, - }, - py3: { - enabled: true, - }, - }, } python_library_host { diff --git a/build/sandbox/nsjail.cfg b/build/sandbox/nsjail.cfg index 509ecc6..5341577 100644 --- a/build/sandbox/nsjail.cfg +++ b/build/sandbox/nsjail.cfg @@ -154,3 +154,10 @@ mount { dst: "/dev/zero" is_bind: true } + +# /dev/stdin used during the creation files in external/cronet +mount { + src: "/proc/self/fd/0" + dst: "/dev/stdin" + is_symlink: true +} diff --git a/build/treble_build/README.md b/build/treble_build/README.md new file mode 100644 index 0000000..1e36bf2 --- /dev/null +++ b/build/treble_build/README.md @@ -0,0 +1,38 @@ +# treble_build + +## Description +Set of tools to run against the Android source tree and build graph. +In order to run the application it must be built via **m treble_build** +this will also create the needed build graph that the tool uses. + +## Basic Commands +- treble_build -h +- treble_build [host, paths, query] [target...] + + +### host +treble_build host + +Report the projects required to build the host tools. + +### paths +treble_build [-build] paths [-1] -repo project:sha [-repo project:sha...] + +For a given set of commits (project:sha), get the corresponding source +files. Translate the source files into a set of build outputs using the +path (-1) or paths command. If the build flag is given build the build +target closest to the source files. + +### query +treble_build query -repo project:sha [-repo project:sha...] + +For a given set of commits (project:sha), get the corresponding source +files. Translate the source files into a set of inputs and outputs. + +### report +By default a report is generated for all above commands, extra targets can +be included in the report by adding to the end of the command line. + +See treble_build -h for options controlling report data. + + diff --git a/build/treble_build/app/Android.bp b/build/treble_build/app/Android.bp new file mode 100644 index 0000000..39071d9 --- /dev/null +++ b/build/treble_build/app/Android.bp @@ -0,0 +1,15 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "treble_report_app", + srcs: [ + "git.go", + "build.go", + "repo.go", + "report.go", + ], + pkgPath: "tools/treble/build/report/app", + pluginFor: ["soong_build"], +} diff --git a/build/treble_build/app/build.go b/build/treble_build/app/build.go new file mode 100644 index 0000000..8b78e23 --- /dev/null +++ b/build/treble_build/app/build.go @@ -0,0 +1,60 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package app + +// Query +type BuildQuery struct { + Target string `json:"target"` + Inputs []string `json:"inputs"` + Outputs []string `json:"outputs"` +} + +// Input +type BuildInput struct { + Target string `json:"target"` + Files []string `json:"files"` +} + +// Commands +type BuildCommand struct { + Target string `json:"target"` + Cmds []string `json:"cmds"` +} + +// Path +type BuildPath struct { + Target string `json:"target"` + Dependency string `json:"dependency"` + Paths []string `json:paths"` +} + +// Build target +type BuildTarget struct { + Name string `json:"name"` // Target name + Steps int `json:"build_steps"` // Number of steps to build target + FileCount int `json:"files"` // Number of input files for a target + Projects map[string]*GitProject `json:"projects"` // Inputs projects/files of a target +} + +// Build command result +type BuildCmdResult struct { + Name string `json:"name"` + Output []string `json:"output"` + Success bool `json:"success"` +} + +// Build dependencies +type BuildDeps struct { + Targets map[string][]string `json:"targets"` +} diff --git a/build/treble_build/app/git.go b/build/treble_build/app/git.go new file mode 100644 index 0000000..7cd21a4 --- /dev/null +++ b/build/treble_build/app/git.go @@ -0,0 +1,73 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +// GIT diff +type GitDiff struct { + AddedLines int `json:"added_lines"` + DeletedLines int `json:"deleted_lines"` + BinaryDiff bool `json:"binary_diff"` +} + +// GIT tree object (files,dirs...) +type GitTreeObj struct { + Permissions string `json:"permissions"` + Type string `json:"type"` + Sha string `json:"sha"` + Filename string `json:"filename"` + BranchDiff *GitDiff `json:"branch_diff"` +} + +// GitProject +type GitProject struct { + RepoDir string `json:"repo_dir"` // Relative directory within repo + WorkDir string `json:"working_dir"` // Working directory + GitDir string `json:"git_dir"` // GIT directory + Remote string `json:"remote"` // Remote Name + RemoteUrl string `json:"remote_url"` // Remote URL + Revision string `json:"revision"` // Revision (SHA) + Files map[string]*GitTreeObj `json:"files"` // Files within the project +} + +type GitCommitFileType int + +const ( + GitFileAdded GitCommitFileType = iota + GitFileModified + GitFileRemoved +) + +type GitCommitFile struct { + Filename string `json:"filename"` + Type GitCommitFileType `json:"type"` +} + +// Git commit +type GitCommit struct { + Sha string `json:"sha"` + Files []GitCommitFile `json:"files"` +} + +func (t GitCommitFileType) String() string { + switch t { + case GitFileModified: + return "M" + case GitFileAdded: + return "A" + case GitFileRemoved: + return "R" + } + return "" +} diff --git a/build/treble_build/app/repo.go b/build/treble_build/app/repo.go new file mode 100644 index 0000000..f1903f2 --- /dev/null +++ b/build/treble_build/app/repo.go @@ -0,0 +1,55 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package app + +import ( + "encoding/xml" + "io/ioutil" +) + +type RepoRemote struct { + Name string `xml:"name,attr"` + Revision string `xml:"fetch,attr"` +} +type RepoDefault struct { + Remote string `xml:"remote,attr"` + Revision string `xml:"revision,attr"` +} +type RepoProject struct { + Groups string `xml:"groups,attr"` + Name string `xml:"name,attr"` + Revision string `xml:"revision,attr"` + Path string `xml:"path,attr"` + Remote *string `xml:"remote,attr"` +} +type RepoManifest struct { + XMLName xml.Name `xml:"manifest"` + Remotes []RepoRemote `xml:"remote"` + Default RepoDefault `xml:"default"` + Projects []RepoProject `xml:"project"` +} + +// Parse a repo manifest file +func ParseXml(filename string) (*RepoManifest, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + v := &RepoManifest{} + err = xml.Unmarshal(data, &v) + if err != nil { + return nil, err + } + return v, nil +} diff --git a/build/treble_build/app/report.go b/build/treble_build/app/report.go new file mode 100644 index 0000000..bc9c7ab --- /dev/null +++ b/build/treble_build/app/report.go @@ -0,0 +1,50 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package app + +// Report request structure +type ReportRequest struct { + Targets []string `json:"targets"` // Targets +} + +// Report response data +type Report struct { + Targets map[string]*BuildTarget `json:"targets"` // Build target data +} + +// Host tool report response data +type HostReport struct { + Path string `json:"path"` // Path to find host tools + SymLinks int `json:"sym_links"` // Number of symlinks found + Targets []string `json:"targets"` // Target for tools found +} + +// Project level commit +type ProjectCommit struct { + Project string `json:"project"` // Project + Revision string `json:"revision"` // Revision +} + +// Query request +type QueryRequest struct { + Files []string `json:"files"` // Files to resolve +} + +// Output response +type QueryResponse struct { + InputFiles []string `json:"input_files"` // Input files found + OutputFiles []string `json:"output_files"` // Output files found + UnknownFiles []string `json:"unknown_files,omitempty"` // Unknown files +} diff --git a/build/treble_build/cmd/Android.bp b/build/treble_build/cmd/Android.bp new file mode 100644 index 0000000..3a1df69 --- /dev/null +++ b/build/treble_build/cmd/Android.bp @@ -0,0 +1,18 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +blueprint_go_binary { + name: "treble_build", + srcs: [ + "host.go", + "main.go", + "paths.go", + "query.go", + ], + deps: [ + "treble_report_app", + "treble_report_module", + "treble_report_local", + ], +} diff --git a/build/treble_build/cmd/host.go b/build/treble_build/cmd/host.go new file mode 100644 index 0000000..0b8e65f --- /dev/null +++ b/build/treble_build/cmd/host.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "path/filepath" + + "tools/treble/build/report/report" +) + +type hostReport struct { + toolPath string +} + +// Determine host tools +func (h *hostReport) Run(ctx context.Context, rtx *report.Context, rsp *response) error { + var err error + rsp.Host, err = report.ResolveHostTools(ctx, h.toolPath) + if err != nil { + return err + } + rsp.Targets = append(rsp.Targets, rsp.Host.Targets...) + return nil +} + +func (h *hostReport) PrintText(w io.Writer, rsp *response, verbose bool) { + if rsp.Host != nil { + // Get the unique number of inputs + hostSourceFileMap := make(map[string]bool) + hostSourceProjectMap := make(map[string]bool) + + for _, t := range rsp.Host.Targets { + // Find target in report + if bt, exists := rsp.Report.Targets[t]; exists { + for name, proj := range bt.Projects { + hostSourceProjectMap[name] = true + for f := range proj.Files { + hostSourceFileMap[filepath.Join(name, f)] = true + } + } + // Remove the target from being printed + delete(rsp.Report.Targets, t) + } + } + + fmt.Fprintln(w, " Host Tools") + fmt.Fprintf(w, " %-20s : %s\n", "Directory", rsp.Host.Path) + fmt.Fprintf(w, " %-20s : %d\n", "Tools", len(rsp.Host.Targets)) + fmt.Fprintf(w, " %-20s : %d\n", "Prebuilts", rsp.Host.SymLinks) + fmt.Fprintf(w, " %-20s : %d\n", "Inputs", len(hostSourceFileMap)) + fmt.Fprintf(w, " %-20s : %d\n", "Projects", len(hostSourceProjectMap)) + + if verbose { + for proj, _ := range hostSourceProjectMap { + fmt.Fprintf(w, " %s\n", proj) + } + } + } + +} diff --git a/build/treble_build/cmd/main.go b/build/treble_build/cmd/main.go new file mode 100644 index 0000000..cced51d --- /dev/null +++ b/build/treble_build/cmd/main.go @@ -0,0 +1,355 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "runtime" + "strings" + "time" + + "tools/treble/build/report/app" + "tools/treble/build/report/local" + "tools/treble/build/report/report" +) + +type Build interface { + Build(ctx context.Context, target string) *app.BuildCmdResult +} + +type tool interface { + Run(ctx context.Context, rtx *report.Context, rsp *response) error + PrintText(w io.Writer, rsp *response, verbose bool) +} +type repoFlags []app.ProjectCommit + +func (r *repoFlags) Set(value string) error { + commit := app.ProjectCommit{} + items := strings.Split(value, ":") + if len(items) > 2 { + return (errors.New("Invalid repo value expected (proj:sha) format")) + } + commit.Project = items[0] + if len(items) > 1 { + commit.Revision = items[1] + } + *r = append(*r, commit) + return nil +} +func (r *repoFlags) String() string { + items := []string{} + for _, fl := range *r { + items = append(items, fmt.Sprintf("%s:%s", fl.Project, fl.Revision)) + } + return strings.Join(items, " ") +} + +var ( + // Common flags + ninjaDbPtr = flag.String("ninja", local.DefNinjaDb(), "Set the .ninja file to use when building metrics") + ninjaExcPtr = flag.String("ninja_cmd", local.DefNinjaExc(), "Set the ninja executable") + ninjaTimeoutStr = flag.String("ninja_timeout", local.DefaultNinjaTimeout, "Default ninja timeout") + buildTimeoutStr = flag.String("build_timeout", local.DefaultNinjaBuildTimeout, "Default build timeout") + manifestPtr = flag.String("manifest", local.DefManifest(), "Set the location of the manifest file") + upstreamPtr = flag.String("upstream", "", "Upstream branch to compare files against") + repoBasePtr = flag.String("repo_base", local.DefRepoBase(), "Set the repo base directory") + workerCountPtr = flag.Int("worker_count", runtime.NumCPU(), "Number of worker routines") + buildWorkerCountPtr = flag.Int("build_worker_count", local.MaxNinjaCliWorkers, "Number of build worker routines") + clientServerPtr = flag.Bool("client_server", false, "Run client server mode") + buildPtr = flag.Bool("build", false, "Build targets") + jsonPtr = flag.Bool("json", false, "Print json data") + verbosePtr = flag.Bool("v", false, "Print verbose text data") + outputPtr = flag.String("o", "", "Output to file") + projsPtr = flag.Bool("projects", false, "Include project repo data") + + hostFlags = flag.NewFlagSet("host", flag.ExitOnError) + queryFlags = flag.NewFlagSet("query", flag.ExitOnError) + pathsFlags = flag.NewFlagSet("paths", flag.ExitOnError) +) + +// Add profiling data +type profTime struct { + Description string `json:"description"` + DurationSecs float64 `json:"duration"` +} + +type commit struct { + Project app.ProjectCommit `json:"project"` + Commit *app.GitCommit `json:"commit"` +} + +// Use one structure for output for now +type response struct { + Commits []commit `json:"commits,omitempty"` + Inputs []string `json:"files,omitempty"` + BuildFiles []*app.BuildCmdResult `json:"build_files,omitempty"` + Targets []string `json:"targets,omitempty"` + Report *app.Report `json:"report,omitempty"` + + // Subcommand data + Query *app.QueryResponse `json:"query,omitempty"` + Paths []*app.BuildPath `json:"build_paths,omitempty"` + Host *app.HostReport `json:"host,omitempty"` + Projects map[string]*app.GitProject `json:"projects,omitempty"` + // Profile data + Profile []*profTime `json:"profile"` +} + +func main() { + startTime := time.Now() + ctx := context.Background() + rsp := &response{} + + var addProfileData = func(desc string) { + rsp.Profile = append(rsp.Profile, &profTime{Description: desc, DurationSecs: time.Since(startTime).Seconds()}) + startTime = time.Now() + } + flag.Parse() + + ninjaTimeout, err := time.ParseDuration(*ninjaTimeoutStr) + if err != nil { + log.Fatalf("Invalid ninja timeout %s", *ninjaTimeoutStr) + } + + buildTimeout, err := time.ParseDuration(*buildTimeoutStr) + if err != nil { + log.Fatalf("Invalid build timeout %s", *buildTimeoutStr) + } + + subArgs := flag.Args() + defBuildTarget := "droid" + log.SetFlags(log.LstdFlags | log.Llongfile) + + ninja := local.NewNinjaCli(*ninjaExcPtr, *ninjaDbPtr, ninjaTimeout, buildTimeout, *clientServerPtr) + + if *clientServerPtr { + ninjaServ := local.NewNinjaServer(*ninjaExcPtr, *ninjaDbPtr) + defer ninjaServ.Kill() + go func() { + + ninjaServ.Start(ctx) + }() + if err := ninja.WaitForServer(ctx, int(ninjaTimeout.Seconds())); err != nil { + log.Fatalf("Failed to connect to server") + } + } + rtx := &report.Context{ + RepoBase: *repoBasePtr, + Repo: &report.RepoMan{}, + Build: ninja, + Project: local.NewGitCli(), + WorkerCount: *workerCountPtr, + BuildWorkerCount: *buildWorkerCountPtr, + } + + var subcommand tool + var commits repoFlags + if len(subArgs) > 0 { + switch subArgs[0] { + case "host": + hostToolPathPtr := hostFlags.String("hostbin", local.DefHostBinPath(), "Set the output directory for host tools") + hostFlags.Parse(subArgs[1:]) + + subcommand = &hostReport{toolPath: *hostToolPathPtr} + rsp.Targets = hostFlags.Args() + + case "query": + queryFlags.Var(&commits, "repo", "Repo:SHA to query") + queryFlags.Parse(subArgs[1:]) + subcommand = &queryReport{} + rsp.Targets = queryFlags.Args() + + case "paths": + pathsFlags.Var(&commits, "repo", "Repo:SHA to build") + singlePathPtr := pathsFlags.Bool("1", false, "Get single path to output target") + pathsFlags.Parse(subArgs[1:]) + + subcommand = &pathsReport{build_target: defBuildTarget, single: *singlePathPtr} + + rsp.Inputs = pathsFlags.Args() + + default: + rsp.Targets = subArgs + } + } + addProfileData("Init") + rtx.ResolveProjectMap(ctx, *manifestPtr, *upstreamPtr) + addProfileData("Project Map") + + // Add project to output if requested + if *projsPtr == true { + rsp.Projects = make(map[string]*app.GitProject) + for k, p := range rtx.Info.ProjMap { + rsp.Projects[k] = p.GitProj + } + } + + // Resolve any commits + if len(commits) > 0 { + log.Printf("Resolving %s", commits.String()) + for _, c := range commits { + commit := commit{Project: c} + info, files, err := report.ResolveCommit(ctx, rtx, &c) + if err != nil { + log.Fatalf("Failed to resolve commit %s:%s", c.Project, c.Revision) + } + commit.Commit = info + rsp.Commits = append(rsp.Commits, commit) + + // Add files to list of inputs + rsp.Inputs = append(rsp.Inputs, files...) + } + addProfileData("Commit Resolution") + } + + // Run any sub tools + if subcommand != nil { + if err := subcommand.Run(ctx, rtx, rsp); err != nil { + log.Fatal(err) + } + addProfileData(subArgs[0]) + } + + buildErrors := 0 + if *buildPtr { + // Only support default builder (non server-client) + builder := local.NewNinjaCli(local.DefNinjaExc(), *ninjaDbPtr, ninjaTimeout, buildTimeout, false /*clientMode*/) + for _, t := range rsp.Targets { + log.Printf("Building %s\n", t) + res := builder.Build(ctx, t) + addProfileData(fmt.Sprintf("Build %s", t)) + log.Printf("%s\n", res.Output) + if res.Success != true { + buildErrors++ + } + rsp.BuildFiles = append(rsp.BuildFiles, res) + } + } + + // Generate report + log.Printf("Generating report for targets %s", rsp.Targets) + req := &app.ReportRequest{Targets: rsp.Targets} + rsp.Report, err = report.RunReport(ctx, rtx, req) + addProfileData("Report") + if err != nil { + log.Fatal(fmt.Sprintf("Report failure <%s>", err)) + } + + if *jsonPtr { + b, _ := json.MarshalIndent(rsp, "", "\t") + if *outputPtr == "" { + os.Stdout.Write(b) + } else { + os.WriteFile(*outputPtr, b, 0644) + } + } else { + if *outputPtr == "" { + printTextReport(os.Stdout, subcommand, rsp, *verbosePtr) + } else { + file, err := os.Create(*outputPtr) + if err != nil { + log.Fatalf("Failed to create output file %s (%s)", *outputPtr, err) + } + w := bufio.NewWriter(file) + printTextReport(w, subcommand, rsp, *verbosePtr) + w.Flush() + } + + } + + if buildErrors > 0 { + log.Fatal(fmt.Sprintf("Failed to build %d targets", buildErrors)) + } +} + +func printTextReport(w io.Writer, subcommand tool, rsp *response, verbose bool) { + fmt.Fprintln(w, "Metric Report") + if subcommand != nil { + subcommand.PrintText(w, rsp, verbose) + } + + if len(rsp.Commits) > 0 { + fmt.Fprintln(w, "") + fmt.Fprintln(w, " Commit Results") + for _, c := range rsp.Commits { + fmt.Fprintf(w, " %-120s : %s\n", c.Project.Project, c.Project.Revision) + fmt.Fprintf(w, " SHA : %s\n", c.Commit.Sha) + fmt.Fprintf(w, " Files : \n") + for _, f := range c.Commit.Files { + fmt.Fprintf(w, " %s %s\n", f.Type.String(), f.Filename) + } + } + } + if len(rsp.BuildFiles) > 0 { + fmt.Fprintln(w, "") + fmt.Fprintln(w, " Build Files") + for _, b := range rsp.BuildFiles { + fmt.Fprintf(w, " %-120s : %t \n", b.Name, b.Success) + } + } + + targetPrint := func(target *app.BuildTarget) { + fmt.Fprintf(w, " %-20s : %s\n", "Name", target.Name) + fmt.Fprintf(w, " %-20s : %d\n", "Build Steps", target.Steps) + fmt.Fprintf(w, " %-20s \n", "Inputs") + fmt.Fprintf(w, " %-20s : %d\n", "Files", target.FileCount) + fmt.Fprintf(w, " %-20s : %d\n", "Projects", len(target.Projects)) + fmt.Fprintln(w) + for name, proj := range target.Projects { + forkCount := 0 + for _, file := range proj.Files { + if file.BranchDiff != nil { + forkCount++ + } + } + fmt.Fprintf(w, " %-120s : %d ", name, len(proj.Files)) + if forkCount != 0 { + fmt.Fprintf(w, " (%d)\n", forkCount) + } else { + fmt.Fprintf(w, " \n") + } + + if verbose { + for _, file := range proj.Files { + var fork string + if file.BranchDiff != nil { + fork = fmt.Sprintf("(%d+ %d-)", file.BranchDiff.AddedLines, file.BranchDiff.DeletedLines) + } + fmt.Fprintf(w, " %-20s %s\n", fork, file.Filename) + } + + } + } + + } + fmt.Fprintln(w, " Targets") + for _, t := range rsp.Report.Targets { + targetPrint(t) + } + + fmt.Fprintln(w, " Run Times") + for _, p := range rsp.Profile { + fmt.Fprintf(w, " %-30s : %f secs\n", p.Description, p.DurationSecs) + } + +} diff --git a/build/treble_build/cmd/paths.go b/build/treble_build/cmd/paths.go new file mode 100644 index 0000000..4c28d4f --- /dev/null +++ b/build/treble_build/cmd/paths.go @@ -0,0 +1,64 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io" + "log" + + "tools/treble/build/report/report" +) + +type pathsReport struct { + build_target string // Target used to filter build request + single bool // Get single path +} + +func (p pathsReport) Run(ctx context.Context, rtx *report.Context, rsp *response) error { + + log.Printf("Resolving paths for %s (single : %v)\n", rsp.Inputs, p.single) + rsp.Paths = report.RunPaths(ctx, rtx, p.build_target, p.single, rsp.Inputs) + + // The path is returned in an array in the form [build_target, path_stop1,...,path_stopN,source_file] + // Choose the closest build target (path_stopN) to the source file to build to reduce the amount that + // is built. + const buildPathIndex = 2 + build_targets := make(map[string]bool) + for _, path := range rsp.Paths { + // Default to build closest build target + if len(path.Paths) > buildPathIndex { + build_targets[path.Paths[len(path.Paths)-buildPathIndex]] = true + } + } + for b := range build_targets { + rsp.Targets = append(rsp.Targets, b) + } + + return nil +} + +func (h *pathsReport) PrintText(w io.Writer, rsp *response, verbose bool) { + if len(rsp.Paths) > 0 { + fmt.Fprintln(w, " Paths") + for _, p := range rsp.Paths { + // Provide path from target to dependency with the + // path length, since target and dependency are in the + // path subtract them out from length + fmt.Fprintf(w, " %s..(%d)..%-s\n", p.Target, len(p.Paths)-2, p.Dependency) + } + } +} diff --git a/build/treble_build/cmd/query.go b/build/treble_build/cmd/query.go new file mode 100644 index 0000000..e4c7f0f --- /dev/null +++ b/build/treble_build/cmd/query.go @@ -0,0 +1,44 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "io" + "log" + + "tools/treble/build/report/app" + "tools/treble/build/report/report" +) + +// Command arguments +type queryReport struct { +} + +// Run query +func (o queryReport) Run(ctx context.Context, rtx *report.Context, rsp *response) error { + var err error + log.Printf("Querying files %s\n", rsp.Inputs) + req := &app.QueryRequest{Files: rsp.Inputs} + rsp.Query, err = report.RunQuery(ctx, rtx, req) + if err != nil { + return err + } + + return nil + +} +func (h *queryReport) PrintText(w io.Writer, rsp *response, verbose bool) { +} diff --git a/build/treble_build/local/Android.bp b/build/treble_build/local/Android.bp new file mode 100644 index 0000000..ffd2781 --- /dev/null +++ b/build/treble_build/local/Android.bp @@ -0,0 +1,22 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "treble_report_local", + srcs: [ + "cmd.go", + "defaults.go", + "git.go", + "ninja.go", + ], + deps: [ + "treble_report_app", + ], + testSrcs: [ + "git_test.go", + "ninja_test.go", + ], + pkgPath: "tools/treble/build/report/local", + pluginFor: ["soong_build"], +} diff --git a/build/treble_build/local/cmd.go b/build/treble_build/local/cmd.go new file mode 100644 index 0000000..3fc24c8 --- /dev/null +++ b/build/treble_build/local/cmd.go @@ -0,0 +1,98 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "os/exec" + "time" +) + +// Run the input command via pipe with given arguments, stdout of the pipe is passed to input parser +// argument. +func runPipe(ctx context.Context, timeout time.Duration, cmdName string, args []string, parser func(r io.Reader)) (err error, stdErr string) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, cmdName, args[0:]...) + errorBuf := bytes.Buffer{} + cmd.Stderr = &errorBuf + stdout, err := cmd.StdoutPipe() + if err != nil { + return err, errorBuf.String() + } + + if err = cmd.Start(); err != nil { + return err, errorBuf.String() + } + parser(stdout) + if err = cmd.Wait(); err != nil { + return err, errorBuf.String() + } + return nil, "" +} + +// Run input command, stdout is passed via out parameter to user, if error the stderr is provided via +// stdErr string to the user. +func run(ctx context.Context, timeout time.Duration, cmdName string, args []string) (out *bytes.Buffer, err error, stdErr string) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, cmdName, args[0:]...) + errorBuf := bytes.Buffer{} + outputBuf := bytes.Buffer{} + cmd.Stderr = &errorBuf + cmd.Stdout = &outputBuf + if err = cmd.Run(); err != nil { + return nil, err, errorBuf.String() + } + + return &outputBuf, nil, "" +} + +// lineScanner +// +// Map output lines to strings, with expected number of +// lines +type lineScanner struct { + Lines []string +} + +// Parse into lines +func (l *lineScanner) Parse(s *bufio.Scanner) error { + i := 0 + for s.Scan() { + if i < len(l.Lines) { + l.Lines[i] = s.Text() + } else { + i++ + break + } + i++ + } + if i != len(l.Lines) { + return errors.New("cmd: incorrect number of lines") + } + return nil +} + +func newLineScanner(numLines int) *lineScanner { + out := &lineScanner{Lines: make([]string, numLines)} + return (out) +} diff --git a/build/treble_build/local/defaults.go b/build/treble_build/local/defaults.go new file mode 100644 index 0000000..53befe4 --- /dev/null +++ b/build/treble_build/local/defaults.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "os" + "path/filepath" +) + +func DefNinjaDb() string { + dbs, _ := filepath.Glob("out/combined-*.ninja") + def_db := "" + if len(dbs) > 0 { + def_db = dbs[0] + } + return def_db +} + +func DefNinjaExc() string { + return "prebuilts/build-tools/linux-x86/bin/ninja" +} +func DefManifest() string { + return ".repo/manifests/default.xml" +} +func DefHostBinPath() string { + return "out/host/linux-x86/bin" +} +func DefRepoBase() string { + ret, _ := os.Getwd() + return ret +} diff --git a/build/treble_build/local/git.go b/build/treble_build/local/git.go new file mode 100644 index 0000000..2793042 --- /dev/null +++ b/build/treble_build/local/git.go @@ -0,0 +1,248 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +// +// Command line implementation of Git interface +// + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "tools/treble/build/report/app" +) + +// Separate out the executable to allow tests to override the results +type gitExec interface { + ProjectInfo(ctx context.Context, gitDir, workDir string) (out *bytes.Buffer, err error) + RemoteUrl(ctx context.Context, gitDir, workDir, remote string) (*bytes.Buffer, error) + Tree(ctx context.Context, gitDir, workDir, revision string) (*bytes.Buffer, error) + CommitInfo(ctx context.Context, gitDir, workDir, revision string) (*bytes.Buffer, error) + DiffBranches(ctx context.Context, gitDir, workDir, upstream, sha string) (*bytes.Buffer, error) +} + +type gitCli struct { + git gitExec // Git executable +} + +// Create GIT project based on input parameters +func (cli gitCli) Project(ctx context.Context, path, gitDir, remote, revision string) (*app.GitProject, error) { + workDir := path + // Set defaults + if remote == "" { + remote = "origin" + } + if gitDir == "" { + gitDir = ".git" + } + + if raw, err := cli.git.ProjectInfo(ctx, gitDir, workDir); err == nil { + topLevel, projRevision, err := parseProjectInfo(raw) + if err == nil { + // Update work dir to use absolute path + workDir = topLevel + if revision == "" { + revision = projRevision + } + } + } + // Create project to use to run commands + out := &app.GitProject{ + RepoDir: path, + WorkDir: workDir, + GitDir: gitDir, + Remote: remote, + Revision: revision, + Files: make(map[string]*app.GitTreeObj)} + + // Remote URL + if raw, err := cli.git.RemoteUrl(ctx, gitDir, workDir, remote); err == nil { + url, err := parseRemoteUrl(raw) + if err == nil { + out.RemoteUrl = url + } + } + + return out, nil +} + +// Get all files in the repository if, upstream branch is provided mark which files differ from upstream +func (cli gitCli) PopulateFiles(ctx context.Context, proj *app.GitProject, upstream string) error { + if raw, err := cli.git.Tree(ctx, proj.GitDir, proj.WorkDir, proj.Revision); err == nil { + lsFiles, err := parseLsTree(raw) + if err == nil { + for _, file := range lsFiles { + proj.Files[file.Filename] = file + } + } + if upstream != "" { + + if diff, err := cli.git.DiffBranches(ctx, proj.GitDir, proj.WorkDir, upstream, proj.Revision); err == nil { + if diffFiles, err := parseBranchDiff(diff); err == nil { + for f, d := range diffFiles { + if file, exists := proj.Files[f]; exists { + file.BranchDiff = d + } + } + } + } + + } + } + return nil +} + +// Get the commit information associated with the input sha +func (cli gitCli) CommitInfo(ctx context.Context, proj *app.GitProject, sha string) (*app.GitCommit, error) { + if sha == "" { + sha = "HEAD" + } + raw, err := cli.git.CommitInfo(ctx, proj.GitDir, proj.WorkDir, sha) + + if err != nil { + return nil, err + } + return parseCommitInfo(raw) +} + +// parse rev-parse +func parseProjectInfo(data *bytes.Buffer) (topLevel string, revision string, err error) { + s := bufio.NewScanner(data) + scanner := newLineScanner(2) + if err = scanner.Parse(s); err != nil { + return "", "", err + } + return scanner.Lines[0], scanner.Lines[1], nil + +} + +// parse remote get-url +func parseRemoteUrl(data *bytes.Buffer) (url string, err error) { + s := bufio.NewScanner(data) + scanner := newLineScanner(1) + if err = scanner.Parse(s); err != nil { + return "", err + } + return scanner.Lines[0], nil + +} + +// parse ls-tree +func parseLsTree(data *bytes.Buffer) ([]*app.GitTreeObj, error) { + out := []*app.GitTreeObj{} + s := bufio.NewScanner(data) + for s.Scan() { + obj := &app.GitTreeObj{} + // TODO + // Filename could contain a <space> as quotepath is turned off, truncating the name here + fmt.Sscanf(s.Text(), "%s %s %s %s", &obj.Permissions, &obj.Type, &obj.Sha, &obj.Filename) + out = append(out, obj) + } + return out, nil +} + +// parse branch diff (diff --num-stat) +func parseBranchDiff(data *bytes.Buffer) (map[string]*app.GitDiff, error) { + out := make(map[string]*app.GitDiff) + s := bufio.NewScanner(data) + for s.Scan() { + d := &app.GitDiff{} + var fname, added, deleted string + _, err := fmt.Sscanf(s.Text(), "%s %s %s", &added, &deleted, &fname) + if err == nil { + if added == "-" || deleted == "-" { + d.BinaryDiff = true + } else { + d.AddedLines, _ = strconv.Atoi(added) + d.DeletedLines, _ = strconv.Atoi(deleted) + } + } + out[fname] = d + } + return out, nil +} + +// parse commit diff-tree +func parseCommitInfo(data *bytes.Buffer) (*app.GitCommit, error) { + out := &app.GitCommit{Files: []app.GitCommitFile{}} + s := bufio.NewScanner(data) + first := true + for s.Scan() { + if first { + out.Sha = s.Text() + } else { + file := app.GitCommitFile{} + t := "" + fmt.Sscanf(s.Text(), "%s %s", &t, &file.Filename) + switch t { + case "M": + file.Type = app.GitFileModified + case "A": + file.Type = app.GitFileAdded + case "R": + file.Type = app.GitFileRemoved + } + out.Files = append(out.Files, file) + } + first = false + } + return out, nil +} + +// Command line git +type gitCmd struct { + cmd string // GIT executable + timeout time.Duration // Timeout for commands +} + +// Run git command in working directory +func (git *gitCmd) runDirCmd(ctx context.Context, gitDir string, workDir string, args []string) (*bytes.Buffer, error) { + gitArgs := append([]string{"--git-dir", gitDir, "-C", workDir}, args...) + out, err, _ := run(ctx, git.timeout, git.cmd, gitArgs) + if err != nil { + return nil, errors.New(fmt.Sprintf("Failed to run %s %s [error %s]", git.cmd, strings.Join(gitArgs, " "))) + } + return out, nil +} + +func (git *gitCmd) ProjectInfo(ctx context.Context, gitDir, workDir string) (*bytes.Buffer, error) { + return git.runDirCmd(ctx, gitDir, workDir, []string{"rev-parse", "--show-toplevel", "HEAD"}) +} +func (git *gitCmd) RemoteUrl(ctx context.Context, gitDir, workDir, remote string) (*bytes.Buffer, error) { + return git.runDirCmd(ctx, gitDir, workDir, []string{"remote", "get-url", remote}) +} +func (git *gitCmd) Tree(ctx context.Context, gitDir, workDir, revision string) (*bytes.Buffer, error) { + cmdArgs := []string{"-c", "core.quotepath=off", "ls-tree", "--full-name", revision, "-r", "-t"} + return git.runDirCmd(ctx, gitDir, workDir, cmdArgs) +} +func (git *gitCmd) CommitInfo(ctx context.Context, gitDir, workDir, sha string) (*bytes.Buffer, error) { + cmdArgs := []string{"diff-tree", "-r", "-m", "--name-status", "--root", sha} + return git.runDirCmd(ctx, gitDir, workDir, cmdArgs) +} +func (git *gitCmd) DiffBranches(ctx context.Context, gitDir, workDir, upstream, sha string) (*bytes.Buffer, error) { + cmdArgs := []string{"diff", "--numstat", fmt.Sprintf("%s...%s", upstream, sha)} + return git.runDirCmd(ctx, gitDir, workDir, cmdArgs) +} +func NewGitCli() *gitCli { + cli := &gitCli{git: &gitCmd{cmd: "git", timeout: 100000 * time.Millisecond}} + return cli +} diff --git a/build/treble_build/local/git_test.go b/build/treble_build/local/git_test.go new file mode 100644 index 0000000..d5345ae --- /dev/null +++ b/build/treble_build/local/git_test.go @@ -0,0 +1,169 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "bytes" + "context" + "reflect" + "testing" + + "tools/treble/build/report/app" +) + +// Test cases for local GIT. +type TestCmd struct { + err error + text string +} +type gitTestCli struct { + revParse *TestCmd + remoteUrl *TestCmd + tree *TestCmd + commit *TestCmd + diffBranch *TestCmd +} + +func (g *gitTestCli) ProjectInfo(ctx context.Context, gitDir, workDir string) (*bytes.Buffer, error) { + return bytes.NewBufferString(g.revParse.text), g.revParse.err +} +func (g *gitTestCli) RemoteUrl(ctx context.Context, gitDir, workDir, remote string) (*bytes.Buffer, error) { + return bytes.NewBufferString(g.remoteUrl.text), g.remoteUrl.err +} +func (g *gitTestCli) Tree(ctx context.Context, gitDir, workDir, revision string) (*bytes.Buffer, error) { + return bytes.NewBufferString(g.tree.text), g.tree.err +} +func (g *gitTestCli) CommitInfo(ctx context.Context, gitDir, workDir, sha string) (*bytes.Buffer, error) { + return bytes.NewBufferString(g.commit.text), g.tree.err +} +func (g *gitTestCli) DiffBranches(ctx context.Context, gitDir, workDir, upstream, sha string) (*bytes.Buffer, error) { + return bytes.NewBufferString(g.diffBranch.text), g.tree.err +} + +func Test_git(t *testing.T) { + + type projectTest struct { + revCmd *TestCmd + remoteCmd *TestCmd + treeCmd *TestCmd + res *app.GitProject + } + + type commitTest struct { + sha string + cmd *TestCmd + res *app.GitCommit + } + + tests := []struct { + path string + gitDir string + remote string + revision string + getFiles bool + project projectTest + commit commitTest + }{ + { + path: "work/dir", + gitDir: "", + remote: "origin", + revision: "sha_revision", + getFiles: true, + project: projectTest{ + revCmd: &TestCmd{text: "/abs/path/to/work/dir\nsha_revision\n", err: nil}, + remoteCmd: &TestCmd{text: "http://url/workdir", err: nil}, + treeCmd: &TestCmd{text: "", err: nil}, + res: &app.GitProject{ + RepoDir: "work/dir", + WorkDir: "/abs/path/to/work/dir", + GitDir: ".git", + Remote: "origin", + RemoteUrl: "http://url/workdir", + Revision: "sha_revision", + Files: make(map[string]*app.GitTreeObj)}, + }, + // Test empty commit + commit: commitTest{ + sha: "commit_sha", + cmd: &TestCmd{text: "commit_sha", err: nil}, + res: &app.GitCommit{Sha: "commit_sha", Files: []app.GitCommitFile{}}, + }, + }, + { + path: "work/dir", + gitDir: "", + remote: "origin", + revision: "sha_revision", + getFiles: true, + project: projectTest{ + revCmd: &TestCmd{text: "/abs/path/to/work/dir\nsha_revision\n", err: nil}, + remoteCmd: &TestCmd{text: "http://url/workdir", err: nil}, + treeCmd: &TestCmd{text: "100644 blob 0000000000000000000000000000000000000001 file.1\n", err: nil}, + res: &app.GitProject{ + RepoDir: "work/dir", + WorkDir: "/abs/path/to/work/dir", + GitDir: ".git", + Remote: "origin", + RemoteUrl: "http://url/workdir", + Revision: "sha_revision", + Files: map[string]*app.GitTreeObj{"file.1": &app.GitTreeObj{Permissions: "100644", Type: "blob", + Sha: "0000000000000000000000000000000000000001", Filename: "file.1"}}}, + }, + commit: commitTest{ + sha: "HEAD", + cmd: &TestCmd{text: "sha_for_head\nR removed.1\nA added.1\nM modified.1\n", err: nil}, + res: &app.GitCommit{ + Sha: "sha_for_head", + Files: []app.GitCommitFile{ + {Filename: "removed.1", Type: app.GitFileRemoved}, + {Filename: "added.1", Type: app.GitFileAdded}, + {Filename: "modified.1", Type: app.GitFileModified}, + }, + }, + }, + }, + } + for _, test := range tests { + git := &gitCli{git: &gitTestCli{ + revParse: test.project.revCmd, + remoteUrl: test.project.remoteCmd, + tree: test.project.treeCmd, + commit: test.commit.cmd, + }} + + proj, err := git.Project(nil, test.path, test.gitDir, test.remote, test.revision) + if err != nil { + t.Fatal("Failed to parse project") + } + if test.getFiles { + _ = git.PopulateFiles(nil, proj, "") + } + if !reflect.DeepEqual(*proj, *test.project.res) { + t.Errorf("Project = %+v; want %+v", *proj, *test.project.res) + } + if test.commit.cmd != nil { + c, err := git.CommitInfo(nil, proj, test.commit.sha) + if err != nil { + t.Errorf("Failed to get; %v", test) + } else { + if !reflect.DeepEqual(*c, *test.commit.res) { + t.Errorf("Commit = %v; want %v", c, *test.commit.res) + } + } + } + } + +} diff --git a/build/treble_build/local/ninja.go b/build/treble_build/local/ninja.go new file mode 100644 index 0000000..956d959 --- /dev/null +++ b/build/treble_build/local/ninja.go @@ -0,0 +1,351 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "os/exec" + "strings" + "time" + + "tools/treble/build/report/app" +) + +// Performance degrades running multiple CLIs +const ( + MaxNinjaCliWorkers = 4 + DefaultNinjaTimeout = "100s" + DefaultNinjaBuildTimeout = "30m" +) + +// Separate out the executable to allow tests to override the results +type ninjaExec interface { + Command(ctx context.Context, target string) (*bytes.Buffer, error) + Input(ctx context.Context, target string) (*bytes.Buffer, error) + Query(ctx context.Context, target string) (*bytes.Buffer, error) + Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) + Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) + Deps(ctx context.Context) (*bytes.Buffer, error) + Build(ctx context.Context, target string) (*bytes.Buffer, error) +} + +// Parse data + +// Add all lines to a given array removing any leading whitespace +func linesToArray(s *bufio.Scanner, arr *[]string) { + for s.Scan() { + line := strings.TrimSpace(s.Text()) + *arr = append(*arr, line) + } +} + +// parse -t commands +func parseCommand(target string, data *bytes.Buffer) (*app.BuildCommand, error) { + out := &app.BuildCommand{Target: target, Cmds: []string{}} + s := bufio.NewScanner(data) + // This tool returns all the commands needed to build a target. + // When running against a target like droid the default capacity + // will be overrun. Extend the capacity here. + const capacity = 1024 * 1024 + buf := make([]byte, capacity) + s.Buffer(buf, capacity) + linesToArray(s, &out.Cmds) + return out, nil +} + +// parse -t inputs +func parseInput(target string, data *bytes.Buffer) (*app.BuildInput, error) { + out := &app.BuildInput{Target: target, Files: []string{}} + s := bufio.NewScanner(data) + linesToArray(s, &out.Files) + return out, nil +} + +// parse -t query +func parseQuery(target string, data *bytes.Buffer) (*app.BuildQuery, error) { + out := &app.BuildQuery{Target: target, Inputs: []string{}, Outputs: []string{}} + const ( + unknown = iota + inputs + outputs + ) + state := unknown + s := bufio.NewScanner(data) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + if strings.HasPrefix(line, "input:") { + state = inputs + } else if strings.HasPrefix(line, "outputs:") { + state = outputs + } else { + switch state { + case inputs: + out.Inputs = append(out.Inputs, line) + case outputs: + out.Outputs = append(out.Outputs, line) + } + } + } + return out, nil +} + +// parse -t path +func parsePath(target string, dependency string, data *bytes.Buffer) (*app.BuildPath, error) { + out := &app.BuildPath{Target: target, Dependency: dependency, Paths: []string{}} + s := bufio.NewScanner(data) + linesToArray(s, &out.Paths) + return out, nil +} + +// parse -t paths +func parsePaths(target string, dependency string, data *bytes.Buffer) ([]*app.BuildPath, error) { + out := []*app.BuildPath{} + s := bufio.NewScanner(data) + for s.Scan() { + path := strings.Fields(s.Text()) + out = append(out, &app.BuildPath{Target: target, Dependency: dependency, Paths: path}) + } + return out, nil +} + +// parse build output +func parseBuild(target string, data *bytes.Buffer, success bool) *app.BuildCmdResult { + out := &app.BuildCmdResult{Name: target, Output: []string{}} + s := bufio.NewScanner(data) + out.Success = success + linesToArray(s, &out.Output) + return out +} + +// parse deps command +func parseDeps(data *bytes.Buffer) (*app.BuildDeps, error) { + out := &app.BuildDeps{Targets: make(map[string][]string)} + s := bufio.NewScanner(data) + curTarget := "" + var deps []string + for s.Scan() { + line := strings.TrimSpace(s.Text()) + // Check if it's a new target + tokens := strings.Split(line, ":") + if len(tokens) > 1 { + if curTarget != "" { + out.Targets[curTarget] = deps + } + deps = []string{} + curTarget = tokens[0] + } else if line != "" { + deps = append(deps, line) + } + + } + if curTarget != "" { + out.Targets[curTarget] = deps + } + return out, nil +} + +// +// Command line interface to ninja binary. +// +// This file implements the ninja.Ninja interface by querying +// the build graph via the ninja binary. The mapping between +// the interface and the binary are as follows: +// Command() -t commands +// Input() -t inputs +// Query() -t query +// Path() -t path +// Paths() -t paths +// Deps() -t deps +// +// + +type ninjaCmd struct { + cmd string + db string + + clientMode bool + timeout time.Duration + buildTimeout time.Duration +} + +func (n *ninjaCmd) runTool(ctx context.Context, tool string, targets []string) (out *bytes.Buffer, err error) { + + args := []string{"-f", n.db} + + if n.clientMode { + args = append(args, []string{ + "-t", "client", + "-c", tool}...) + } else { + args = append(args, []string{"-t", tool}...) + } + args = append(args, targets...) + data := []byte{} + err, _ = runPipe(ctx, n.timeout, n.cmd, args, func(r io.Reader) { + data, _ = ioutil.ReadAll(r) + }) + return bytes.NewBuffer(data), err + +} +func (n *ninjaCmd) Command(ctx context.Context, target string) (*bytes.Buffer, error) { + return n.runTool(ctx, "commands", []string{target}) +} +func (n *ninjaCmd) Input(ctx context.Context, target string) (*bytes.Buffer, error) { + return n.runTool(ctx, "inputs", []string{target}) +} +func (n *ninjaCmd) Query(ctx context.Context, target string) (*bytes.Buffer, error) { + return n.runTool(ctx, "query", []string{target}) +} +func (n *ninjaCmd) Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { + return n.runTool(ctx, "path", []string{target, dependency}) +} +func (n *ninjaCmd) Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { + return n.runTool(ctx, "paths", []string{target, dependency}) +} +func (n *ninjaCmd) Deps(ctx context.Context) (*bytes.Buffer, error) { + return n.runTool(ctx, "deps", []string{}) +} + +func (n *ninjaCmd) Build(ctx context.Context, target string) (*bytes.Buffer, error) { + + args := append([]string{ + "-f", n.db, + target}) + data := []byte{} + err, _ := runPipe(ctx, n.buildTimeout, n.cmd, args, func(r io.Reader) { + data, _ = ioutil.ReadAll(r) + }) + + return bytes.NewBuffer(data), err +} + +// Command line ninja +type ninjaCli struct { + n ninjaExec +} + +// ninja -t commands +func (cli *ninjaCli) Command(ctx context.Context, target string) (*app.BuildCommand, error) { + raw, err := cli.n.Command(ctx, target) + if err != nil { + return nil, err + } + return parseCommand(target, raw) +} + +// ninja -t inputs +func (cli *ninjaCli) Input(ctx context.Context, target string) (*app.BuildInput, error) { + raw, err := cli.n.Input(ctx, target) + if err != nil { + return nil, err + } + return parseInput(target, raw) +} + +// ninja -t query +func (cli *ninjaCli) Query(ctx context.Context, target string) (*app.BuildQuery, error) { + raw, err := cli.n.Query(ctx, target) + if err != nil { + return nil, err + } + return parseQuery(target, raw) +} + +// ninja -t path +func (cli *ninjaCli) Path(ctx context.Context, target string, dependency string) (*app.BuildPath, error) { + raw, err := cli.n.Path(ctx, target, dependency) + if err != nil { + return nil, err + } + return parsePath(target, dependency, raw) +} + +// ninja -t paths +func (cli *ninjaCli) Paths(ctx context.Context, target string, dependency string) ([]*app.BuildPath, error) { + raw, err := cli.n.Paths(ctx, target, dependency) + if err != nil { + return nil, err + } + return parsePaths(target, dependency, raw) +} + +// ninja -t deps +func (cli *ninjaCli) Deps(ctx context.Context) (*app.BuildDeps, error) { + raw, err := cli.n.Deps(ctx) + if err != nil { + return nil, err + } + return parseDeps(raw) +} + +// Build given target +func (cli *ninjaCli) Build(ctx context.Context, target string) *app.BuildCmdResult { + raw, err := cli.n.Build(ctx, target) + return parseBuild(target, raw, err == nil) + +} + +// Wait for server +func (cli *ninjaCli) WaitForServer(ctx context.Context, maxTries int) error { + // Wait for server to response to an empty input request + fmt.Printf("Waiting for server.") + for i := 0; i < maxTries; i++ { + _, err := cli.Input(ctx, "") + if err == nil { + fmt.Printf("\nConnected\n") + return nil + } + fmt.Printf(".") + time.Sleep(time.Second) + } + fmt.Printf(" failed\n") + return errors.New("Failed to connect") +} +func NewNinjaCli(cmd string, db string, timeout, buildTimeout time.Duration, client bool) *ninjaCli { + cli := &ninjaCli{n: &ninjaCmd{cmd: cmd, db: db, timeout: timeout, buildTimeout: buildTimeout, clientMode: client}} + return cli +} + +type ninjaServer struct { + cmdName string + db string + ctx *exec.Cmd +} + +// Run server +func (srv *ninjaServer) Start(ctx context.Context) error { + args := []string{"-f", srv.db, "-t", "server"} + srv.ctx = exec.CommandContext(ctx, srv.cmdName, args[0:]...) + err := srv.ctx.Start() + if err != nil { + return err + } + srv.ctx.Wait() + return nil +} +func (srv *ninjaServer) Kill() { + if srv.ctx != nil { + srv.ctx.Process.Kill() + } +} +func NewNinjaServer(cmd string, db string) *ninjaServer { + return &ninjaServer{cmdName: cmd, db: db} +} diff --git a/build/treble_build/local/ninja_test.go b/build/treble_build/local/ninja_test.go new file mode 100644 index 0000000..40e2043 --- /dev/null +++ b/build/treble_build/local/ninja_test.go @@ -0,0 +1,216 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "bytes" + "context" + "reflect" + "testing" + + "tools/treble/build/report/app" +) + +type ninjaTest struct { + command *TestCmd + input *TestCmd + query *TestCmd + path *TestCmd + paths *TestCmd + deps *TestCmd + build *TestCmd +} + +func (n *ninjaTest) Command(ctx context.Context, target string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.command.text), n.command.err +} +func (n *ninjaTest) Input(ctx context.Context, target string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.input.text), n.input.err +} +func (n *ninjaTest) Query(ctx context.Context, target string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.query.text), n.query.err +} +func (n *ninjaTest) Path(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.path.text), n.path.err +} +func (n *ninjaTest) Paths(ctx context.Context, target string, dependency string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.paths.text), n.paths.err +} +func (n *ninjaTest) Deps(ctx context.Context) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.deps.text), n.deps.err +} +func (n *ninjaTest) Build(ctx context.Context, target string) (*bytes.Buffer, error) { + return bytes.NewBufferString(n.build.text), n.build.err +} + +func Test_ninja(t *testing.T) { + type commandTest struct { + cmd *TestCmd + res *app.BuildCommand + } + type queryTest struct { + cmd *TestCmd + res *app.BuildQuery + } + type inputTest struct { + cmd *TestCmd + res *app.BuildInput + } + type pathTest struct { + cmd *TestCmd + res *app.BuildPath + } + type pathsTest struct { + cmd *TestCmd + res []*app.BuildPath + } + type depsTest struct { + cmd *TestCmd + res *app.BuildDeps + } + type buildTest struct { + cmd *TestCmd + res *app.BuildCmdResult + } + tests := []struct { + target string + dependency string + command commandTest + query queryTest + input inputTest + path pathTest + paths pathsTest + deps depsTest + build buildTest + }{ + { + target: "test", + dependency: "dependency", + command: commandTest{ + cmd: &TestCmd{text: " cmd1\ncmd2\n cmd3\n", err: nil}, + res: &app.BuildCommand{Target: "test", Cmds: []string{"cmd1", "cmd2", "cmd3"}}}, + query: queryTest{ + cmd: &TestCmd{text: "input:\ninfile\noutputs:\noutfile\n", err: nil}, + res: &app.BuildQuery{Target: "test", Inputs: []string{"infile"}, Outputs: []string{"outfile"}}}, + input: inputTest{ + cmd: &TestCmd{text: "file1\nfile2\nfile3\nfile4\nfile5\n", err: nil}, + res: &app.BuildInput{Target: "test", Files: []string{"file1", "file2", "file3", "file4", "file5"}}, + }, + path: pathTest{ + cmd: &TestCmd{text: "test\nmid1\nmid2\nmid3\ndependency\n", err: nil}, + res: &app.BuildPath{Target: "test", Dependency: "dependency", + Paths: []string{"test", "mid1", "mid2", "mid3", "dependency"}}, + }, + paths: pathsTest{ + cmd: &TestCmd{text: "test mid1 mid2 mid3 dependency\ntest mid4 dependency\n", err: nil}, + res: []*app.BuildPath{ + &app.BuildPath{Target: "test", Dependency: "dependency", Paths: []string{"test", "mid1", "mid2", "mid3", "dependency"}}, + &app.BuildPath{Target: "test", Dependency: "dependency", Paths: []string{"test", "mid4", "dependency"}}, + }, + }, + deps: depsTest{ + cmd: &TestCmd{text: "some/build/library.so: #deps1\n dependentFile1.S\n dependentFile2.S\nsome/build/library2.so: #deps1\n dependentFile1.S\n dependentFile3.S\n"}, + res: &app.BuildDeps{Targets: map[string][]string{ + "some/build/library.so": []string{"dependentFile1.S", "dependentFile2.S"}, + "some/build/library2.so": []string{"dependentFile1.S", "dependentFile3.S"}, + }, + }, + }, + build: buildTest{ + cmd: &TestCmd{text: "", err: nil}, + res: &app.BuildCmdResult{Name: "test", Output: []string{}, Success: true}}, + }, + } + for _, test := range tests { + + exec := &ninjaTest{ + command: test.command.cmd, + query: test.query.cmd, + input: test.input.cmd, + path: test.path.cmd, + paths: test.paths.cmd, + deps: test.deps.cmd, + build: test.build.cmd, + } + n := &ninjaCli{n: exec} + + if test.command.cmd != nil { + if res, err := n.Command(nil, test.target); err != nil { + t.Errorf("Command error %s", err) + } else { + if !reflect.DeepEqual(*res, *test.command.res) { + t.Errorf("Command result %v; want %v", *res, *test.command.res) + } + } + } + if test.query.cmd != nil { + if res, err := n.Query(nil, test.target); err != nil { + t.Errorf("Query error %s", err) + } else { + if !reflect.DeepEqual(*res, *test.query.res) { + t.Errorf("Query result %v; want %v", *res, *test.query.res) + } + } + + } + if test.input.cmd != nil { + if res, err := n.Input(nil, test.target); err != nil { + t.Errorf("Input error %s", err) + } else { + if !reflect.DeepEqual(*res, *test.input.res) { + t.Errorf("Input result %v; want %v", *res, *test.input.res) + } + } + + } + if test.path.cmd != nil { + if res, err := n.Path(nil, test.target, test.dependency); err != nil { + t.Errorf("Path error %s", err) + } else { + if !reflect.DeepEqual(*res, *test.path.res) { + t.Errorf("Path result %v; want %v", *res, *test.path.res) + } + } + + } + if test.paths.cmd != nil { + if res, err := n.Paths(nil, test.target, test.dependency); err != nil { + t.Errorf("Paths error %s", err) + } else { + if !reflect.DeepEqual(res, test.paths.res) { + t.Errorf("Paths result %v; want %v", res, test.paths.res) + } + } + + } + if test.deps.cmd != nil { + if res, err := n.Deps(nil); err != nil { + t.Errorf("Deps error %s", err) + } else { + if !reflect.DeepEqual(res, test.deps.res) { + t.Errorf("Deps result %v; want %v", res, test.deps.res) + } + } + + } + if test.build.cmd != nil { + res := n.Build(nil, test.target) + if !reflect.DeepEqual(*res, *test.build.res) { + t.Errorf("Build result %+v; want %+v", *res, *test.build.res) + } + + } + } +} diff --git a/build/treble_build/report/Android.bp b/build/treble_build/report/Android.bp new file mode 100644 index 0000000..33fd57d --- /dev/null +++ b/build/treble_build/report/Android.bp @@ -0,0 +1,20 @@ +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "treble_report_module", + srcs: [ + "dependencies.go", + "build.go", + "projects.go", + "run.go", + "types.go", + ], + testSrcs: [ + "report_test.go", + ], + deps: ["treble_report_app"], + pkgPath: "tools/treble/build/report/report", + pluginFor: ["soong_build"], +} diff --git a/build/treble_build/report/build.go b/build/treble_build/report/build.go new file mode 100644 index 0000000..cc0c9bf --- /dev/null +++ b/build/treble_build/report/build.go @@ -0,0 +1,182 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "fmt" + "sync" + + "tools/treble/build/report/app" +) + +// Channel data structures, include explicit error field to reply to each input +type buildTargetData struct { + input *app.BuildInput + buildSteps int + error bool +} +type buildSourceData struct { + source string + query *app.BuildQuery + error bool +} +type buildPathData struct { + filename string + path *app.BuildPath + error bool +} + +// +// create build target from using repo data +// +func createBuildTarget(ctx context.Context, rtx *Context, buildTarget *buildTargetData) *app.BuildTarget { + out := &app.BuildTarget{Name: buildTarget.input.Target, + Steps: buildTarget.buildSteps, + Projects: make(map[string]*app.GitProject), + FileCount: len(buildTarget.input.Files), + } + + for _, f := range buildTarget.input.Files { + proj, buildFile := lookupProjectFile(ctx, rtx, f) + if buildFile != nil { + if buildProj, exists := out.Projects[proj.Name]; exists { + buildProj.Files[buildFile.Filename] = buildFile + } else { + out.Projects[proj.Name] = + &app.GitProject{ + RepoDir: proj.GitProj.RepoDir, + WorkDir: proj.GitProj.WorkDir, + GitDir: proj.GitProj.GitDir, + Remote: proj.GitProj.Remote, + RemoteUrl: proj.GitProj.RemoteUrl, + Revision: proj.GitProj.Revision, + Files: map[string]*app.GitTreeObj{buildFile.Filename: buildFile}} + } + } + } + return (out) +} + +// Setup routines to resolve target names to app.BuildInput objects +func targetResolvers(ctx context.Context, rtx *Context) (chan string, chan *buildTargetData) { + var wg sync.WaitGroup + inChan := make(chan string) + outChan := make(chan *buildTargetData) + for i := 0; i < rtx.BuildWorkerCount; i++ { + wg.Add(1) + go func() { + for targetName := range inChan { + var buildSteps int + cmds, err := rtx.Build.Command(ctx, targetName) + if err == nil { + buildSteps = len(cmds.Cmds) + } + input, err := rtx.Build.Input(ctx, targetName) + if input == nil { + fmt.Printf("Failed to get input %s (%s)\n", targetName, err) + } else { + outChan <- &buildTargetData{input: input, buildSteps: buildSteps, error: err != nil} + } + } + wg.Done() + }() + } + go func() { + wg.Wait() + close(outChan) + }() + + return inChan, outChan +} + +// +// Setup routines to resolve build input targets to BuildTarget +func resolveBuildInputs(ctx context.Context, rtx *Context, inChan chan *buildTargetData) chan *app.BuildTarget { + var wg sync.WaitGroup + outChan := make(chan *app.BuildTarget) + for i := 0; i < rtx.BuildWorkerCount; i++ { + wg.Add(1) + go func() { + for buildTarget := range inChan { + outChan <- createBuildTarget(ctx, rtx, buildTarget) + } + wg.Done() + }() + } + go func() { + wg.Wait() + close(outChan) + }() + return outChan +} + +// Setup routines to resolve source file to query +func queryResolvers(ctx context.Context, rtx *Context) (chan string, chan *buildSourceData) { + var wg sync.WaitGroup + inChan := make(chan string) + outChan := make(chan *buildSourceData) + for i := 0; i < rtx.BuildWorkerCount; i++ { + wg.Add(1) + go func() { + for srcName := range inChan { + query, err := rtx.Build.Query(ctx, srcName) + outChan <- &buildSourceData{source: srcName, query: query, error: err != nil} + } + wg.Done() + }() + } + go func() { + wg.Wait() + close(outChan) + }() + + return inChan, outChan +} + +// Setup routines to resolve paths +func pathsResolvers(ctx context.Context, rtx *Context, target string, singlePath bool) (chan string, chan *buildPathData) { + var wg sync.WaitGroup + inChan := make(chan string) + outChan := make(chan *buildPathData) + for i := 0; i < rtx.BuildWorkerCount; i++ { + wg.Add(1) + go func() { + for dep := range inChan { + if singlePath { + path, err := rtx.Build.Path(ctx, target, dep) + outChan <- &buildPathData{filename: dep, path: path, error: err != nil} + } else { + paths, err := rtx.Build.Paths(ctx, target, dep) + if err != nil { + outChan <- &buildPathData{filename: dep, path: nil, error: true} + } else { + for _, path := range paths { + + outChan <- &buildPathData{filename: dep, path: path, error: false} + } + } + } + } + wg.Done() + }() + } + go func() { + wg.Wait() + close(outChan) + }() + + return inChan, outChan +} diff --git a/build/treble_build/report/dependencies.go b/build/treble_build/report/dependencies.go new file mode 100644 index 0000000..f2b21b5 --- /dev/null +++ b/build/treble_build/report/dependencies.go @@ -0,0 +1,40 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + + "tools/treble/build/report/app" +) + +type BuildDependencies interface { + Command(ctx context.Context, target string) (*app.BuildCommand, error) + Input(ctx context.Context, target string) (*app.BuildInput, error) + Query(ctx context.Context, target string) (*app.BuildQuery, error) + Path(ctx context.Context, target string, dependency string) (*app.BuildPath, error) + Paths(ctx context.Context, target string, dependency string) ([]*app.BuildPath, error) + Deps(ctx context.Context) (*app.BuildDeps, error) +} + +type ProjectDependencies interface { + Project(ctx context.Context, path string, gitDir string, remote string, revision string) (*app.GitProject, error) + PopulateFiles(ctx context.Context, proj *app.GitProject, upstream string) error + CommitInfo(ctx context.Context, proj *app.GitProject, sha string) (*app.GitCommit, error) +} + +type RepoDependencies interface { + Manifest(filename string) (*app.RepoManifest, error) +} diff --git a/build/treble_build/report/projects.go b/build/treble_build/report/projects.go new file mode 100644 index 0000000..eaec5bf --- /dev/null +++ b/build/treble_build/report/projects.go @@ -0,0 +1,178 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "tools/treble/build/report/app" +) + +// +// Repo and project related functions +// +type project struct { + Name string // Name + GitProj *app.GitProject // Git project data +} + +var unknownProject = &project{Name: "unknown", GitProj: &app.GitProject{}} + +// Convert repo project to project with source files and revision +// information +func resolveProject(ctx context.Context, repoProj *app.RepoProject, remote *app.RepoRemote, proj ProjectDependencies, getFiles bool, upstreamBranch string) *project { + + path := repoProj.Path + if path == "" { + path = repoProj.Name + } + gitDir := "" + if strings.HasPrefix(path, "overlays/") { + // Assume two levels of overlay path (overlay/XYZ) + path = strings.Join(strings.Split(path, "/")[2:], "/") + // The overlays .git symbolic links are not mapped correctly + // into the jails. Resolve them here, inside the nsjail the + // absolute path for all git repos will be in the form of + // /src/.git/ + symlink, _ := os.Readlink(filepath.Join(path, ".git")) + parts := strings.Split(symlink, "/") + repostart := 0 + for ; repostart < len(parts); repostart++ { + if parts[repostart] != ".." { + if repostart > 1 { + repostart-- + parts[repostart] = "/src" + } + break + } + } + gitDir = filepath.Join(parts[repostart:]...) + + } + gitProj, err := proj.Project(ctx, path, gitDir, remote.Name, repoProj.Revision) + if err != nil { + return nil + } + out := &project{Name: repoProj.Name, GitProj: gitProj} + if getFiles { + _ = proj.PopulateFiles(ctx, gitProj, upstreamBranch) + } + return out +} + +// Get the build file for a given filename, this is a two step lookup. +// First find the project associated with the file via the file cache, +// then resolve the file via the project found. +// +// Most files will be relative paths from the repo workspace +func lookupProjectFile(ctx context.Context, rtx *Context, filename string) (*project, *app.GitTreeObj) { + if proj, exists := rtx.Info.FileCache[filename]; exists { + repoName := (filename)[len(proj.GitProj.RepoDir)+1:] + if gitObj, exists := proj.GitProj.Files[repoName]; exists { + return proj, gitObj + } + return proj, nil + } else { + // Try resolving any symlinks + if realpath, err := filepath.EvalSymlinks(filename); err == nil { + if realpath != filename { + return lookupProjectFile(ctx, rtx, realpath) + } + } + + if strings.HasPrefix(filename, rtx.RepoBase) { + // Some dependencies pick up the full path try stripping out + relpath := (filename)[len(rtx.RepoBase):] + return lookupProjectFile(ctx, rtx, relpath) + } + } + return unknownProject, &app.GitTreeObj{Filename: filename, Sha: ""} +} + +// Create a mapping of projects from the input source manifest +func resolveProjectMap(ctx context.Context, rtx *Context, manifestFile string, getFiles bool, upstreamBranch string) *ProjectInfo { + // Parse the manifest file + manifest, err := rtx.Repo.Manifest(manifestFile) + if err != nil { + return nil + } + info := &ProjectInfo{} + // Create map of remotes + remotes := make(map[string]*app.RepoRemote) + var defRemotePtr *app.RepoRemote + for i, _ := range manifest.Remotes { + remotes[manifest.Remotes[i].Name] = &manifest.Remotes[i] + } + + defRemotePtr, exists := remotes[manifest.Default.Remote] + if !exists { + fmt.Printf("Failed to find default remote") + } + info.FileCache = make(map[string]*project) + info.ProjMap = make(map[string]*project) + + var wg sync.WaitGroup + projChan := make(chan *project) + repoChan := make(chan *app.RepoProject) + + for i := 0; i < rtx.WorkerCount; i++ { + wg.Add(1) + go func() { + for repoProj := range repoChan { + remotePtr := defRemotePtr + if manifest.Projects[i].Remote != nil { + remotePtr = remotes[*manifest.Projects[i].Remote] + } + proj := resolveProject(ctx, repoProj, remotePtr, rtx.Project, getFiles, upstreamBranch) + if proj != nil { + projChan <- proj + } else { + projChan <- &project{Name: repoProj.Name} + } + } + wg.Done() + }() + } + go func() { + wg.Wait() + close(projChan) + }() + go func() { + for i, _ := range manifest.Projects { + repoChan <- &manifest.Projects[i] + } + close(repoChan) + }() + for r := range projChan { + if r.GitProj != nil { + info.ProjMap[r.Name] = r + if len(r.GitProj.Files) > 0 { + for n := range r.GitProj.Files { + info.FileCache[filepath.Join(r.GitProj.RepoDir, n)] = r + } + + } + + } else { + fmt.Printf("Failed to resolve %s\n", r.Name) + } + } + return info +} diff --git a/build/treble_build/report/report_test.go b/build/treble_build/report/report_test.go new file mode 100644 index 0000000..7aa3891 --- /dev/null +++ b/build/treble_build/report/report_test.go @@ -0,0 +1,266 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "reflect" + "strconv" + "testing" + + "tools/treble/build/report/app" +) + +type reportTest struct { + manifest *app.RepoManifest + commands map[string]*app.BuildCommand + inputs map[string]*app.BuildInput + queries map[string]*app.BuildQuery + paths map[string]map[string]*app.BuildPath + multipaths map[string]map[string][]*app.BuildPath + projects map[string]*app.GitProject + commits map[*app.GitProject]map[string]*app.GitCommit + + deps *app.BuildDeps + projectCommits map[string]int +} + +func (r *reportTest) Manifest(filename string) (*app.RepoManifest, error) { + var err error + out := r.manifest + if out == nil { + err = errors.New(fmt.Sprintf("No manifest named %s", filename)) + } + return r.manifest, err +} +func (r *reportTest) Command(ctx context.Context, target string) (*app.BuildCommand, error) { + var err error + out := r.commands[target] + if out == nil { + err = errors.New(fmt.Sprintf("No command for target %s", target)) + } + return out, err +} + +func (r *reportTest) Input(ctx context.Context, target string) (*app.BuildInput, error) { + var err error + out := r.inputs[target] + if out == nil { + err = errors.New(fmt.Sprintf("No inputs for target %s", target)) + } + return out, err +} + +func (r *reportTest) Query(ctx context.Context, target string) (*app.BuildQuery, error) { + var err error + out := r.queries[target] + if out == nil { + err = errors.New(fmt.Sprintf("No queries for target %s", target)) + } + return out, err +} + +func (r *reportTest) Path(ctx context.Context, target string, dependency string) (*app.BuildPath, error) { + return r.paths[target][dependency], nil +} + +func (r *reportTest) Paths(ctx context.Context, target string, dependency string) ([]*app.BuildPath, error) { + return r.multipaths[target][dependency], nil +} + +func (r *reportTest) Deps(ctx context.Context) (*app.BuildDeps, error) { + return r.deps, nil +} +func (r *reportTest) Project(ctx context.Context, path string, gitDir string, remote string, revision string) (*app.GitProject, error) { + var err error + out := r.projects[path] + if out == nil { + err = errors.New(fmt.Sprintf("No projects for target %s", path)) + } + return out, err +} +func (r *reportTest) PopulateFiles(ctx context.Context, proj *app.GitProject, upstream string) error { + return nil +} +func (r *reportTest) CommitInfo(ctx context.Context, proj *app.GitProject, sha string) (*app.GitCommit, error) { + var err error + out := r.commits[proj][sha] + if out == nil { + err = errors.New(fmt.Sprintf("No commit for sha %s", sha)) + } + return out, err +} + +// Helper routine used in test function to create array of unique names +func createStrings(name string, count int) []string { + var out []string + for i := 0; i < count; i++ { + out = append(out, name+strconv.Itoa(i)) + } + return out +} + +// Project names used in tests +func projName(i int) string { + return "proj." + strconv.Itoa(i) +} + +func fileName(i int) (filename string, sha string) { + iString := strconv.Itoa(i) + return "source." + iString, "sha." + iString +} +func createFile(i int) *app.GitTreeObj { + fname, sha := fileName(i) + return &app.GitTreeObj{Permissions: "100644", Type: "blob", Filename: fname, Sha: sha} +} +func createProject(name string) *app.GitProject { + return &app.GitProject{ + RepoDir: name, WorkDir: name, GitDir: ".git", Remote: "origin", + RemoteUrl: "origin_url", Revision: name + "_sha", + Files: make(map[string]*app.GitTreeObj)} + +} + +// Create basic test data for given inputs +func createTest(projCount int, fileCount int) *reportTest { + test := &reportTest{ + manifest: &app.RepoManifest{ + Remotes: []app.RepoRemote{{Name: "remote1", Revision: "revision_1"}}, + Default: app.RepoDefault{Remote: "remote1", Revision: "revision_2"}, + Projects: []app.RepoProject{}, + }, + commands: map[string]*app.BuildCommand{}, + inputs: map[string]*app.BuildInput{}, + queries: map[string]*app.BuildQuery{}, + projects: map[string]*app.GitProject{}, + commits: map[*app.GitProject]map[string]*app.GitCommit{}, + } + + // Create projects with files + for i := 0; i <= projCount; i++ { + name := projName(i) + + proj := createProject(name) + + for i := 0; i <= fileCount; i++ { + treeObj := createFile(i) + proj.Files[treeObj.Filename] = treeObj + + } + test.projects[name] = proj + test.manifest.Projects = append(test.manifest.Projects, + app.RepoProject{Groups: "group", Name: name, Revision: "sha", Path: name}) + + } + return test +} + +func Test_report(t *testing.T) { + + test := createTest(10, 20) + + // Test cases will specify input file by project and file index + type inputFile struct { + proj int + file int + } + + targetDefs := []struct { + name string // Target name + cmds int // Number of build steps + inputTargets int // Number of input targets + outputTargets int // Number of output targets + inputFiles []inputFile // Input files for target + }{ + { + name: "target", + cmds: 7, + inputTargets: 4, + outputTargets: 7, + inputFiles: []inputFile{{proj: 0, file: 1}, {proj: 1, file: 0}}, + }, + { + name: "target2", + cmds: 0, + inputTargets: 0, + outputTargets: 0, + inputFiles: []inputFile{{proj: 0, file: 1}, {proj: 0, file: 2}, {proj: 1, file: 0}}, + }, + { + name: "null_target", + cmds: 0, + inputTargets: 0, + outputTargets: 0, + inputFiles: []inputFile{}, + }, + } + + // Create target data based on definitions + var targets []string + + // Build expected output while creating the targets + resTargets := make(map[string]*app.BuildTarget) + + for _, target := range targetDefs { + + res := &app.BuildTarget{Name: target.name, + Steps: target.cmds, + FileCount: len(target.inputFiles), + Projects: make(map[string]*app.GitProject), + } + + // Add files to the build target + var inputFiles []string + for _, in := range target.inputFiles { + // Get project by name + pName := projName(in.proj) + bf := createFile(in.file) + p := test.projects[pName] + + inputFiles = append(inputFiles, + fmt.Sprintf("%s/%s", p.WorkDir, bf.Filename)) + + if _, exists := res.Projects[pName]; !exists { + res.Projects[pName] = createProject(pName) + } + res.Projects[pName].Files[bf.Filename] = bf + } + + // Create test data + test.commands[target.name] = &app.BuildCommand{Target: target.name, Cmds: createStrings("cmd.", target.cmds)} + test.inputs[target.name] = &app.BuildInput{Target: target.name, Files: inputFiles} + test.queries[target.name] = &app.BuildQuery{ + Target: target.name, + Inputs: createStrings("target.in.", target.inputTargets), + Outputs: createStrings("target.out.", target.outputTargets)} + + targets = append(targets, target.name) + resTargets[res.Name] = res + } + + rtx := &Context{RepoBase: "/src", Repo: test, Build: test, Project: test, WorkerCount: 1, BuildWorkerCount: 1} + rtx.ResolveProjectMap(nil, "test_file", "") + req := &app.ReportRequest{Targets: targets} + rsp, err := RunReport(nil, rtx, req) + if err != nil { + t.Errorf("Failed to run report for request %+v", req) + } else { + if !reflect.DeepEqual(rsp.Targets, resTargets) { + t.Errorf("Got targets %+v, expected %+v", rsp.Targets, resTargets) + } + } +} diff --git a/build/treble_build/report/run.go b/build/treble_build/report/run.go new file mode 100644 index 0000000..fd03088 --- /dev/null +++ b/build/treble_build/report/run.go @@ -0,0 +1,191 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io/fs" + "path/filepath" + + "tools/treble/build/report/app" +) + +// Find all binary executables under the given directory along with the number +// of symlinks +// +func binaryExecutables(ctx context.Context, dir string, recursive bool) ([]string, int, error) { + var files []string + numSymLinks := 0 + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + if info, err := d.Info(); err == nil { + if info.Mode()&0111 != 0 { + files = append(files, path) + } + if d.Type()&fs.ModeSymlink != 0 { + numSymLinks++ + } + } + } else { + if !recursive { + if path != dir { + return filepath.SkipDir + } + } + } + return nil + }) + + return files, numSymLinks, err +} + +// Resolve the manifest +func (rtx *Context) ResolveProjectMap(ctx context.Context, manifest string, upstreamBranch string) { + if rtx.Info == nil { + rtx.Info = resolveProjectMap(ctx, rtx, manifest, true, upstreamBranch) + } +} + +// Find host tools +func ResolveHostTools(ctx context.Context, hostToolPath string) (*app.HostReport, error) { + out := &app.HostReport{Path: hostToolPath} + out.Targets, out.SymLinks, _ = binaryExecutables(ctx, hostToolPath, true) + return out, nil +} + +// Run reports + +// +// Run report request +// +// Setup routines to: +// - resolve the manifest projects +// - resolve build queries +// +// Once the manifest projects have been resolved the build +// queries can be fully resolved +// +func RunReport(ctx context.Context, rtx *Context, req *app.ReportRequest) (*app.Report, error) { + inChan, targetCh := targetResolvers(ctx, rtx) + go func() { + for i, _ := range req.Targets { + inChan <- req.Targets[i] + } + close(inChan) + }() + + // Resolve the build inputs into build target projects + buildTargetChan := resolveBuildInputs(ctx, rtx, targetCh) + + out := &app.Report{Targets: make(map[string]*app.BuildTarget)} + for bt := range buildTargetChan { + out.Targets[bt.Name] = bt + } + + return out, nil +} + +// Resolve commit into git commit info +func ResolveCommit(ctx context.Context, rtx *Context, commit *app.ProjectCommit) (*app.GitCommit, []string, error) { + if proj, exists := rtx.Info.ProjMap[commit.Project]; exists { + info, err := rtx.Project.CommitInfo(ctx, proj.GitProj, commit.Revision) + files := []string{} + if err == nil { + for _, f := range info.Files { + if f.Type != app.GitFileRemoved { + files = append(files, filepath.Join(proj.GitProj.RepoDir, f.Filename)) + } + } + } + return info, files, err + } + return nil, nil, errors.New(fmt.Sprintf("Unknown project %s", commit.Project)) + +} + +// Run query report based on the input request. +// +// For each input file query the target and +// create a set of the inputs and outputs associated +// with all the input files. +// +// +func RunQuery(ctx context.Context, rtx *Context, req *app.QueryRequest) (*app.QueryResponse, error) { + inChan, queryCh := queryResolvers(ctx, rtx) + + go func() { + // Convert source files to outputs + for _, target := range req.Files { + inChan <- target + } + close(inChan) + }() + + inFiles := make(map[string]bool) + outFiles := make(map[string]bool) + unknownSrcFiles := make(map[string]bool) + for result := range queryCh { + if result.error { + unknownSrcFiles[result.source] = true + } else { + for _, outFile := range result.query.Outputs { + outFiles[outFile] = true + } + for _, inFile := range result.query.Inputs { + inFiles[inFile] = true + } + + } + } + + out := &app.QueryResponse{} + for k, _ := range outFiles { + out.OutputFiles = append(out.OutputFiles, k) + } + for k, _ := range inFiles { + out.InputFiles = append(out.InputFiles, k) + } + for k, _ := range unknownSrcFiles { + out.UnknownFiles = append(out.UnknownFiles, k) + } + + return out, nil +} + +// Get paths +func RunPaths(ctx context.Context, rtx *Context, target string, singlePath bool, files []string) []*app.BuildPath { + out := []*app.BuildPath{} + inChan, pathCh := pathsResolvers(ctx, rtx, target, singlePath) + // Convert source files to outputs + go func() { + for _, f := range files { + inChan <- f + } + close(inChan) + }() + + for result := range pathCh { + if !result.error { + out = append(out, result.path) + } + } + return out + +} diff --git a/build/treble_build/report/types.go b/build/treble_build/report/types.go new file mode 100644 index 0000000..8610d43 --- /dev/null +++ b/build/treble_build/report/types.go @@ -0,0 +1,45 @@ +// Copyright 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "tools/treble/build/report/app" +) + +type RepoMan struct { +} + +func (r *RepoMan) Manifest(filename string) (*app.RepoManifest, error) { + return app.ParseXml(filename) +} + +// Project information containing a map of projects, this also contains a +// map between a source file and the project it belongs to +// allowing a quicker lookup of source file to project +type ProjectInfo struct { + ProjMap map[string]*project // Map project name to project + FileCache map[string]*project // Map source files to project +} + +// Report context +type Context struct { + RepoBase string // Absolute path to repo base + Repo RepoDependencies // Repo interface + Build BuildDependencies // Build interface + Project ProjectDependencies // Project interface + WorkerCount int // Number of worker threads + BuildWorkerCount int // Number of build worker threads + Info *ProjectInfo // Project information +} diff --git a/fetcher/Android.bp b/fetcher/Android.bp index 787d0b7..654a750 100644 --- a/fetcher/Android.bp +++ b/fetcher/Android.bp @@ -1,30 +1,9 @@ package { - // See: http://go/android-license-faq - // A large-scale-change added 'default_applicable_licenses' to import - // all of the 'license_kinds' from "tools_treble_license" - // to get the below license kinds: - // SPDX-license-identifier-Apache-2.0 - default_applicable_licenses: ["tools_treble_license"], + default_applicable_licenses: ["Android-Apache-2.0"], } -python_defaults { - name: "fetcher-defaults", - version: { - py2: { - enabled: false, - embedded_launcher: false, - }, - py3: { - enabled: true, - embedded_launcher: false, - }, - }, -} - - python_library_host { name: "fetcher-lib", - defaults: ["fetcher-defaults"], srcs: [ "fetcher_lib.py" ], @@ -32,14 +11,12 @@ python_library_host { libs: [ "py-google-api-python-client", "py-oauth2client", - "py-six", ], } python_binary_host { name: "fetcher", main: "fetcher.py", - defaults: ["fetcher-defaults"], srcs: [ "fetcher.py", ], diff --git a/fetcher/fetcher_lib.py b/fetcher/fetcher_lib.py index 9701494..3e6288f 100644 --- a/fetcher/fetcher_lib.py +++ b/fetcher/fetcher_lib.py @@ -15,7 +15,7 @@ if sys.version_info.major == 3: # pylint: disable=import-error,g-bad-import-order,g-import-not-at-top import apiclient from googleapiclient.discovery import build -from six.moves import http_client +from googleapiclient.errors import HttpError import httplib2 from oauth2client.service_account import ServiceAccountCredentials @@ -80,7 +80,7 @@ def _simple_execute(http_request, for _ in range(max_tries): try: return http_request.execute() - except http_client.errors.HttpError as e: + except HttpError as e: last_error = e if e.resp.status in masked_errors: return None @@ -103,7 +103,7 @@ def create_client(http): Returns: An authorized android build api client. """ - return build(serviceName='androidbuildinternal', version='v2beta1', http=http, + return build(serviceName='androidbuildinternal', version='v3', http=http, static_discovery=False) @@ -201,14 +201,12 @@ def list_artifacts(client, regex, **kwargs): """ matching_artifacts = [] kwargs.setdefault('attemptId', 'latest') - regex = re.compile(regex) - req = client.buildartifact().list(**kwargs) + req = client.buildartifact().list(nameRegexp=regex, **kwargs) while req: result = _simple_execute(req) if result and 'artifacts' in result: for a in result['artifacts']: - if regex.match(a['name']): - matching_artifacts.append(a['name']) + matching_artifacts.append(a['name']) req = client.buildartifact().list_next(req, result) return matching_artifacts diff --git a/gki/Android.bp b/gki/Android.bp index d5b886d..29f8077 100644 --- a/gki/Android.bp +++ b/gki/Android.bp @@ -1,29 +1,9 @@ package { - // See: http://go/android-license-faq - // A large-scale-change added 'default_applicable_licenses' to import - // all of the 'license_kinds' from "tools_treble_license" - // to get the below license kinds: - // SPDX-license-identifier-Apache-2.0 - default_applicable_licenses: ["tools_treble_license"], -} - -python_defaults { - name: "repack_gki_defaults", - version: { - py2: { - enabled: false, - embedded_launcher: false, - }, - py3: { - enabled: true, - embedded_launcher: false, - }, - }, + default_applicable_licenses: ["Android-Apache-2.0"], } python_library_host { name: "repack_gki_lib", - defaults: ["repack_gki_defaults"], srcs: [ "repack_gki_lib.py", ], @@ -36,7 +16,6 @@ python_library_host { python_binary_host { name: "repack_gki", main: "repack_gki.py", - defaults: ["repack_gki_defaults"], srcs: [ "repack_gki.py", ], diff --git a/gki/repack_gki.py b/gki/repack_gki.py index 90b632e..19c11cb 100644 --- a/gki/repack_gki.py +++ b/gki/repack_gki.py @@ -17,13 +17,14 @@ def main(): help='JSON keyfile containing credentials. ' '(Default: Use default credential file)') parser.add_argument( - '--ramdisk_build_id', - required=True, - help='Download from the specified build.') + '--bootimg_build_id', help='Download from the specified build.') parser.add_argument( - '--ramdisk_target', - required=True, - help='Name of the ramdisk target from the ramdisk branch.') + '--ramdisk_build_id', help='DEPRECATED. Use --bootimg_build_id instead.') + parser.add_argument( + '--bootimg_target', + help='Name of the bootimg target from the bootimg branch.') + parser.add_argument( + '--ramdisk_target', help='DEPRECATED. Use --bootimg_target instead.') parser.add_argument( '--kernel_build_id', required=True, @@ -50,15 +51,18 @@ def main(): if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) + bootimg_build_id = args.bootimg_build_id or args.ramdisk_build_id + bootimg_target = args.bootimg_target or args.ramdisk_target + with tempfile.TemporaryDirectory() as tmp_bootimg_dir, \ tempfile.TemporaryDirectory() as tmp_kernel_dir: # Fetch boot images. repack_gki_lib.fetch_bootimg( client=client, out_dir=tmp_bootimg_dir, - build_id=args.ramdisk_build_id, + build_id=bootimg_build_id, kernel_version=args.kernel_version, - target=args.ramdisk_target, + target=bootimg_target, ) # Fetch kernel artifacts. @@ -84,6 +88,7 @@ def main(): copy_kernel_file(kernel_dir, 'System.map') copy_kernel_file(kernel_dir, 'abi_symbollist') copy_kernel_file(kernel_dir, 'vmlinux') + copy_kernel_file(kernel_dir, 'vmlinux.symvers') copy_kernel_file(kernel_dir, 'Image', 'kernel-{}'.format(args.kernel_version)) copy_kernel_file(kernel_dir, 'Image.lz4', @@ -99,6 +104,8 @@ def main(): 'kernel-{}-lz4-allsyms'.format(args.kernel_version)) copy_kernel_file(kernel_debug_dir, 'Image.gz', 'kernel-{}-gz-allsyms'.format(args.kernel_version)) + copy_kernel_file(kernel_debug_dir, 'vmlinux', 'vmlinux-allsyms') + copy_kernel_file(kernel_debug_dir, 'vmlinux.symvers', 'vmlinux.symvers-allsyms') # Repack individual boot images using the fetched kernel artifacts, # then save to the out dir. @@ -113,24 +120,20 @@ def main(): args.kernel_version) shutil.copy(img_zip_path, args.out_dir) - # Replace kernels within the target_files.zip and save to the out dir. - # TODO(b/209035444): GSI target_files does not yet include a 5.15 boot.img. - if args.kernel_version != '5.15': - target_files_zip_name = [ - f for f in os.listdir(tmp_bootimg_dir) if '-target_files-' in f - ][0] - target_files_zip_path = os.path.join(tmp_bootimg_dir, target_files_zip_name) - repack_gki_lib.replace_target_files_zip_kernels(target_files_zip_path, - kernel_out_dir, - args.kernel_version) - shutil.copy(target_files_zip_path, args.out_dir) - - # Copy otatools.zip from the ramdisk build, used for GKI signing. + target_files_zip_name = [ + f for f in os.listdir(tmp_bootimg_dir) if '-target_files-' in f + ][0] + target_files_zip_path = os.path.join(tmp_bootimg_dir, target_files_zip_name) + repack_gki_lib.replace_target_files_zip_kernels(target_files_zip_path, + kernel_out_dir, + args.kernel_version) + shutil.copy(target_files_zip_path, args.out_dir) + + # Copy otatools.zip from the bootimg build, used for GKI signing. shutil.copy(os.path.join(tmp_bootimg_dir, 'otatools.zip'), args.out_dir) # Write prebuilt-info.txt using the prebuilt artifact build IDs. data = { - 'ramdisk-build-id': int(args.ramdisk_build_id), 'kernel-build-id': int(args.kernel_build_id), } with open(os.path.join(kernel_out_dir, 'prebuilt-info.txt'), 'w') as f: diff --git a/gki/repack_gki_lib.py b/gki/repack_gki_lib.py index 9051a65..ccabb94 100644 --- a/gki/repack_gki_lib.py +++ b/gki/repack_gki_lib.py @@ -13,7 +13,7 @@ def fetch_bootimg(client, out_dir, build_id, kernel_version, target): client=client, build_id=build_id, target=target, - pattern=r'(gsi_.*-img-.*\.zip|gsi_.*-target_files-.*\.zip|boot-debug-{version}.*\.img|boot-test-harness-{version}.*\.img|otatools.zip)' + pattern=r'(gki_.*-img-.*\.zip|gki_.*-target_files-.*\.zip|otatools.zip)' .format(version=kernel_version), out_dir=out_dir) @@ -29,13 +29,13 @@ def fetch_kernel(client, out_dir, build_id, kernel_target, kernel_debug_target): client=client, build_id=build_id, target=kernel_target, - pattern=r'(Image|Image.lz4|System\.map|abi_symbollist|vmlinux)', + pattern=r'(Image|Image.lz4|System\.map|abi_symbollist|vmlinux|vmlinux.symvers)', out_dir=kernel_dir) fetcher_lib.fetch_artifacts( client=client, build_id=build_id, target=kernel_debug_target, - pattern=r'(Image|Image.lz4|System\.map|abi-generated.xml|abi-full-generated.xml)', + pattern=r'(Image|Image.lz4|System\.map|abi-generated.xml|abi-full-generated.xml|vmlinux|vymlinx.symvers)', out_dir=kernel_debug_dir) print('Compressing kernels') @@ -107,10 +107,6 @@ def repack_bootimgs(bootimg_dir, kernel_dir, kernel_debug_dir): def repack_img_zip(img_zip_path, kernel_dir, kernel_debug_dir, kernel_version): """Repacks boot images within an img.zip archive.""" with tempfile.TemporaryDirectory() as unzip_dir: - # TODO(b/209035444): 5.15 GSI boot.img is not yet available, so reuse 5.10 boot.img - # which should have an identical ramdisk. - if kernel_version == '5.15': - kernel_version = '5.10' pattern = 'boot-{}*'.format(kernel_version) print('Unzipping %s to repack bootimgs' % img_zip_path) cmd = [ diff --git a/split/Android.bp b/split/Android.bp index f35167f..d742681 100644 --- a/split/Android.bp +++ b/split/Android.bp @@ -1,37 +1,10 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package { - // See: http://go/android-license-faq - // A large-scale-change added 'default_applicable_licenses' to import - // all of the 'license_kinds' from "tools_treble_license" - // to get the below license kinds: - // SPDX-license-identifier-Apache-2.0 - default_applicable_licenses: ["tools_treble_license"], + default_applicable_licenses: ["Android-Apache-2.0"], } python_defaults { name: "treble_split_default", pkg_path: "treble/split", - version: { - py2: { - enabled: false, - }, - py3: { - enabled: true, - }, - }, libs: [ "py-setuptools", ], |