diff options
author | Bob Badour <bbadour@google.com> | 2022-10-12 20:10:17 -0700 |
---|---|---|
committer | Bob Badour <bbadour@google.com> | 2022-10-18 16:55:47 -0700 |
commit | dc62de4760f47b65800d2037d6210573316d3151 (patch) | |
tree | d2483d3a5afdd662713ce71d4927e6c8e627d48d | |
parent | 6974223827c369d6380150ce3c822c700b4fc59c (diff) | |
download | build-dc62de4760f47b65800d2037d6210573316d3151.tar.gz |
Refactor projectmetadata into separate package.
Replace regular expressions to extract fields from a text proto with
and actual parsed protobuf.
Refactor TestFS into its own package, and implement StatFS.
Test: m droid dist cts alllicensemetadata
Test: repo forall -c 'echo -n "$REPO_PATH " && $ANDROID_BUILD_TOP/out/host/linux-x86/bin/compliance_checkmetadata . 2>&1' | fgrep -v PASS
Change-Id: Icd17a6a2b6a4e2b6ffded48e964b9c9d6e4d64d6
18 files changed, 1094 insertions, 54 deletions
diff --git a/tools/compliance/Android.bp b/tools/compliance/Android.bp index 225f3a578b..2527df727b 100644 --- a/tools/compliance/Android.bp +++ b/tools/compliance/Android.bp @@ -18,6 +18,17 @@ package { } blueprint_go_binary { + name: "compliance_checkmetadata", + srcs: ["cmd/checkmetadata/checkmetadata.go"], + deps: [ + "compliance-module", + "projectmetadata-module", + "soong-response", + ], + testSrcs: ["cmd/checkmetadata/checkmetadata_test.go"], +} + +blueprint_go_binary { name: "compliance_checkshare", srcs: ["cmd/checkshare/checkshare.go"], deps: [ @@ -156,6 +167,8 @@ bootstrap_go_package { "test_util.go", ], deps: [ + "compliance-test-fs-module", + "projectmetadata-module", "golang-protobuf-proto", "golang-protobuf-encoding-prototext", "license_metadata_proto", diff --git a/tools/compliance/cmd/checkmetadata/checkmetadata.go b/tools/compliance/cmd/checkmetadata/checkmetadata.go new file mode 100644 index 0000000000..c6c84e45a7 --- /dev/null +++ b/tools/compliance/cmd/checkmetadata/checkmetadata.go @@ -0,0 +1,148 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "android/soong/response" + "android/soong/tools/compliance" + "android/soong/tools/compliance/projectmetadata" +) + +var ( + failNoneRequested = fmt.Errorf("\nNo projects requested") +) + +func main() { + var expandedArgs []string + for _, arg := range os.Args[1:] { + if strings.HasPrefix(arg, "@") { + f, err := os.Open(strings.TrimPrefix(arg, "@")) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + respArgs, err := response.ReadRspFile(f) + f.Close() + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + expandedArgs = append(expandedArgs, respArgs...) + } else { + expandedArgs = append(expandedArgs, arg) + } + } + + flags := flag.NewFlagSet("flags", flag.ExitOnError) + + flags.Usage = func() { + fmt.Fprintf(os.Stderr, `Usage: %s {-o outfile} projectdir {projectdir...} + +Tries to open the METADATA.android or METADATA file in each projectdir +reporting any errors on stderr. + +Reports "FAIL" to stdout if any errors found and exits with status 1. + +Otherwise, reports "PASS" and the number of project metadata files +found exiting with status 0. +`, filepath.Base(os.Args[0])) + flags.PrintDefaults() + } + + outputFile := flags.String("o", "-", "Where to write the output. (default stdout)") + + flags.Parse(expandedArgs) + + // Must specify at least one root target. + if flags.NArg() == 0 { + flags.Usage() + os.Exit(2) + } + + if len(*outputFile) == 0 { + flags.Usage() + fmt.Fprintf(os.Stderr, "must specify file for -o; use - for stdout\n") + os.Exit(2) + } else { + dir, err := filepath.Abs(filepath.Dir(*outputFile)) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot determine path to %q: %s\n", *outputFile, err) + os.Exit(1) + } + fi, err := os.Stat(dir) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot read directory %q of %q: %s\n", dir, *outputFile, err) + os.Exit(1) + } + if !fi.IsDir() { + fmt.Fprintf(os.Stderr, "parent %q of %q is not a directory\n", dir, *outputFile) + os.Exit(1) + } + } + + var ofile io.Writer + ofile = os.Stdout + var obuf *bytes.Buffer + if *outputFile != "-" { + obuf = &bytes.Buffer{} + ofile = obuf + } + + err := checkProjectMetadata(ofile, os.Stderr, compliance.FS, flags.Args()...) + if err != nil { + if err == failNoneRequested { + flags.Usage() + } + fmt.Fprintf(os.Stderr, "%s\n", err.Error()) + fmt.Fprintln(ofile, "FAIL") + os.Exit(1) + } + if *outputFile != "-" { + err := os.WriteFile(*outputFile, obuf.Bytes(), 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "could not write output to %q from %q: %s\n", *outputFile, os.Getenv("PWD"), err) + os.Exit(1) + } + } + os.Exit(0) +} + +// checkProjectMetadata implements the checkmetadata utility. +func checkProjectMetadata(stdout, stderr io.Writer, rootFS fs.FS, projects ...string) error { + + if len(projects) < 1 { + return failNoneRequested + } + + // Read the project metadata files from `projects` + ix := projectmetadata.NewIndex(rootFS) + pms, err := ix.MetadataForProjects(projects...) + if err != nil { + return fmt.Errorf("Unable to read project metadata file(s) %q from %q: %w\n", projects, os.Getenv("PWD"), err) + } + + fmt.Fprintf(stdout, "PASS -- parsed %d project metadata files for %d projects\n", len(pms), len(projects)) + return nil +} diff --git a/tools/compliance/cmd/checkmetadata/checkmetadata_test.go b/tools/compliance/cmd/checkmetadata/checkmetadata_test.go new file mode 100644 index 0000000000..cf2090b4cc --- /dev/null +++ b/tools/compliance/cmd/checkmetadata/checkmetadata_test.go @@ -0,0 +1,191 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + + "android/soong/tools/compliance" +) + +func TestMain(m *testing.M) { + // Change into the parent directory before running the tests + // so they can find the testdata directory. + if err := os.Chdir(".."); err != nil { + fmt.Printf("failed to change to testdata directory: %s\n", err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func Test(t *testing.T) { + tests := []struct { + name string + projects []string + expectedStdout string + }{ + { + name: "1p", + projects: []string{"firstparty"}, + expectedStdout: "PASS -- parsed 1 project metadata files for 1 projects", + }, + { + name: "notice", + projects: []string{"notice"}, + expectedStdout: "PASS -- parsed 1 project metadata files for 1 projects", + }, + { + name: "1p+notice", + projects: []string{"firstparty", "notice"}, + expectedStdout: "PASS -- parsed 2 project metadata files for 2 projects", + }, + { + name: "reciprocal", + projects: []string{"reciprocal"}, + expectedStdout: "PASS -- parsed 1 project metadata files for 1 projects", + }, + { + name: "1p+notice+reciprocal", + projects: []string{"firstparty", "notice", "reciprocal"}, + expectedStdout: "PASS -- parsed 3 project metadata files for 3 projects", + }, + { + name: "restricted", + projects: []string{"restricted"}, + expectedStdout: "PASS -- parsed 1 project metadata files for 1 projects", + }, + { + name: "1p+notice+reciprocal+restricted", + projects: []string{ + "firstparty", + "notice", + "reciprocal", + "restricted", + }, + expectedStdout: "PASS -- parsed 4 project metadata files for 4 projects", + }, + { + name: "proprietary", + projects: []string{"proprietary"}, + expectedStdout: "PASS -- parsed 1 project metadata files for 1 projects", + }, + { + name: "1p+notice+reciprocal+restricted+proprietary", + projects: []string{ + "firstparty", + "notice", + "reciprocal", + "restricted", + "proprietary", + }, + expectedStdout: "PASS -- parsed 5 project metadata files for 5 projects", + }, + { + name: "missing1", + projects: []string{"regressgpl1"}, + expectedStdout: "PASS -- parsed 0 project metadata files for 1 projects", + }, + { + name: "1p+notice+reciprocal+restricted+proprietary+missing1", + projects: []string{ + "firstparty", + "notice", + "reciprocal", + "restricted", + "proprietary", + "regressgpl1", + }, + expectedStdout: "PASS -- parsed 5 project metadata files for 6 projects", + }, + { + name: "missing2", + projects: []string{"regressgpl2"}, + expectedStdout: "PASS -- parsed 0 project metadata files for 1 projects", + }, + { + name: "1p+notice+reciprocal+restricted+proprietary+missing1+missing2", + projects: []string{ + "firstparty", + "notice", + "reciprocal", + "restricted", + "proprietary", + "regressgpl1", + "regressgpl2", + }, + expectedStdout: "PASS -- parsed 5 project metadata files for 7 projects", + }, + { + name: "missing2+1p+notice+reciprocal+restricted+proprietary+missing1", + projects: []string{ + "regressgpl2", + "firstparty", + "notice", + "reciprocal", + "restricted", + "proprietary", + "regressgpl1", + }, + expectedStdout: "PASS -- parsed 5 project metadata files for 7 projects", + }, + { + name: "missing2+1p+notice+missing1+reciprocal+restricted+proprietary", + projects: []string{ + "regressgpl2", + "firstparty", + "notice", + "regressgpl1", + "reciprocal", + "restricted", + "proprietary", + }, + expectedStdout: "PASS -- parsed 5 project metadata files for 7 projects", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + + projects := make([]string, 0, len(tt.projects)) + for _, project := range tt.projects { + projects = append(projects, "testdata/"+project) + } + err := checkProjectMetadata(stdout, stderr, compliance.GetFS(""), projects...) + if err != nil { + t.Fatalf("checkmetadata: error = %v, stderr = %v", err, stderr) + return + } + var actualStdout string + for _, s := range strings.Split(stdout.String(), "\n") { + ts := strings.TrimLeft(s, " \t") + if len(ts) < 1 { + continue + } + if len(actualStdout) > 0 { + t.Errorf("checkmetadata: unexpected multiple output lines %q, want %q", actualStdout+"\n"+ts, tt.expectedStdout) + } + actualStdout = ts + } + if actualStdout != tt.expectedStdout { + t.Errorf("checkmetadata: unexpected stdout %q, want %q", actualStdout, tt.expectedStdout) + } + }) + } +} diff --git a/tools/compliance/cmd/testdata/firstparty/METADATA b/tools/compliance/cmd/testdata/firstparty/METADATA new file mode 100644 index 0000000000..62b4481159 --- /dev/null +++ b/tools/compliance/cmd/testdata/firstparty/METADATA @@ -0,0 +1,6 @@ +# Comments are allowed +name: "1ptd" +description: "First Party Test Data" +third_party { + version: "1.0" +} diff --git a/tools/compliance/cmd/testdata/notice/METADATA b/tools/compliance/cmd/testdata/notice/METADATA new file mode 100644 index 0000000000..302dfeb54e --- /dev/null +++ b/tools/compliance/cmd/testdata/notice/METADATA @@ -0,0 +1,6 @@ +# Comments are allowed +name: "noticetd" +description: "Notice Test Data" +third_party { + version: "1.0" +} diff --git a/tools/compliance/cmd/testdata/proprietary/METADATA b/tools/compliance/cmd/testdata/proprietary/METADATA new file mode 100644 index 0000000000..72cc54ab9b --- /dev/null +++ b/tools/compliance/cmd/testdata/proprietary/METADATA @@ -0,0 +1 @@ +# comments are allowed diff --git a/tools/compliance/cmd/testdata/reciprocal/METADATA b/tools/compliance/cmd/testdata/reciprocal/METADATA new file mode 100644 index 0000000000..50cc2ef3ef --- /dev/null +++ b/tools/compliance/cmd/testdata/reciprocal/METADATA @@ -0,0 +1,5 @@ +# Comments are allowed +description: "Reciprocal Test Data" +third_party { + version: "1.0" +} diff --git a/tools/compliance/cmd/testdata/restricted/METADATA b/tools/compliance/cmd/testdata/restricted/METADATA new file mode 100644 index 0000000000..6bcf83f6bb --- /dev/null +++ b/tools/compliance/cmd/testdata/restricted/METADATA @@ -0,0 +1,6 @@ +name { + id: 1 +} +third_party { + version: 2 +} diff --git a/tools/compliance/cmd/testdata/restricted/METADATA.android b/tools/compliance/cmd/testdata/restricted/METADATA.android new file mode 100644 index 0000000000..1142499dee --- /dev/null +++ b/tools/compliance/cmd/testdata/restricted/METADATA.android @@ -0,0 +1,6 @@ +# Comments are allowed +name: "testdata" +description: "Restricted Test Data" +third_party { + version: "1.0" +} diff --git a/tools/compliance/policy_policy_test.go b/tools/compliance/policy_policy_test.go index 94d0be332c..6188eb202d 100644 --- a/tools/compliance/policy_policy_test.go +++ b/tools/compliance/policy_policy_test.go @@ -20,6 +20,8 @@ import ( "sort" "strings" "testing" + + "android/soong/tools/compliance/testfs" ) func TestPolicy_edgeConditions(t *testing.T) { @@ -210,7 +212,7 @@ func TestPolicy_edgeConditions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fs := make(testFS) + fs := make(testfs.TestFS) stderr := &bytes.Buffer{} target := meta[tt.edge.target] + fmt.Sprintf("deps: {\n file: \"%s\"\n", tt.edge.dep) for _, ann := range tt.edge.annotations { diff --git a/tools/compliance/projectmetadata/Android.bp b/tools/compliance/projectmetadata/Android.bp new file mode 100644 index 0000000000..dccff7697f --- /dev/null +++ b/tools/compliance/projectmetadata/Android.bp @@ -0,0 +1,34 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "projectmetadata-module", + srcs: [ + "projectmetadata.go", + ], + deps: [ + "compliance-test-fs-module", + "golang-protobuf-proto", + "golang-protobuf-encoding-prototext", + "project_metadata_proto", + ], + testSrcs: [ + "projectmetadata_test.go", + ], + pkgPath: "android/soong/tools/compliance/projectmetadata", +} diff --git a/tools/compliance/projectmetadata/projectmetadata.go b/tools/compliance/projectmetadata/projectmetadata.go new file mode 100644 index 0000000000..b31413d03a --- /dev/null +++ b/tools/compliance/projectmetadata/projectmetadata.go @@ -0,0 +1,209 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package projectmetadata + +import ( + "fmt" + "io" + "io/fs" + "path/filepath" + "strings" + "sync" + + "android/soong/compliance/project_metadata_proto" + + "google.golang.org/protobuf/encoding/prototext" +) + +var ( + // ConcurrentReaders is the size of the task pool for limiting resource usage e.g. open files. + ConcurrentReaders = 5 +) + +// ProjectMetadata contains the METADATA for a git project. +type ProjectMetadata struct { + proto project_metadata_proto.Metadata + + // project is the path to the directory containing the METADATA file. + project string +} + +// String returns a string representation of the metadata for error messages. +func (pm *ProjectMetadata) String() string { + return fmt.Sprintf("project: %q\n%s", pm.project, pm.proto.String()) +} + +// VersionedName returns the name of the project including the version if any. +func (pm *ProjectMetadata) VersionedName() string { + name := pm.proto.GetName() + if name != "" { + tp := pm.proto.GetThirdParty() + if tp != nil { + version := tp.GetVersion() + if version != "" { + if version[0] == 'v' || version[0] == 'V' { + return name + "_" + version + } else { + return name + "_v_" + version + } + } + } + return name + } + return pm.proto.GetDescription() +} + +// projectIndex describes a project to be read; after `wg.Wait()`, will contain either +// a `ProjectMetadata`, pm (can be nil even without error), or a non-nil `err`. +type projectIndex struct { + project string + pm *ProjectMetadata + err error + done chan struct{} +} + +// finish marks the task to read the `projectIndex` completed. +func (pi *projectIndex) finish() { + close(pi.done) +} + +// wait suspends execution until the `projectIndex` task completes. +func (pi *projectIndex) wait() { + <-pi.done +} + +// Index reads and caches ProjectMetadata (thread safe) +type Index struct { + // projecs maps project name to a wait group if read has already started, and + // to a `ProjectMetadata` or to an `error` after the read completes. + projects sync.Map + + // task provides a fixed-size task pool to limit concurrent open files etc. + task chan bool + + // rootFS locates the root of the file system from which to read the files. + rootFS fs.FS +} + +// NewIndex constructs a project metadata `Index` for the given file system. +func NewIndex(rootFS fs.FS) *Index { + ix := &Index{task: make(chan bool, ConcurrentReaders), rootFS: rootFS} + for i := 0; i < ConcurrentReaders; i++ { + ix.task <- true + } + return ix +} + +// MetadataForProjects returns 0..n ProjectMetadata for n `projects`, or an error. +// Each project that has a METADATA.android or a METADATA file in the root of the project will have +// a corresponding ProjectMetadata in the result. Projects with neither file get skipped. A nil +// result with no error indicates none of the given `projects` has a METADATA file. +// (thread safe -- can be called concurrently from multiple goroutines) +func (ix *Index) MetadataForProjects(projects ...string) ([]*ProjectMetadata, error) { + if ConcurrentReaders < 1 { + return nil, fmt.Errorf("need at least one task in project metadata pool") + } + if len(projects) == 0 { + return nil, nil + } + // Identify the projects that have never been read + projectsToRead := make([]*projectIndex, 0, len(projects)) + projectIndexes := make([]*projectIndex, 0, len(projects)) + for _, p := range projects { + pi, loaded := ix.projects.LoadOrStore(p, &projectIndex{project: p, done: make(chan struct{})}) + if !loaded { + projectsToRead = append(projectsToRead, pi.(*projectIndex)) + } + projectIndexes = append(projectIndexes, pi.(*projectIndex)) + } + // findMeta locates and reads the appropriate METADATA file, if any. + findMeta := func(pi *projectIndex) { + <-ix.task + defer func() { + ix.task <- true + pi.finish() + }() + + // Support METADATA.android for projects that already have a different sort of METADATA file. + path := filepath.Join(pi.project, "METADATA.android") + fi, err := fs.Stat(ix.rootFS, path) + if err == nil { + if fi.Mode().IsRegular() { + ix.readMetadataFile(pi, path) + return + } + } + // No METADATA.android try METADATA file. + path = filepath.Join(pi.project, "METADATA") + fi, err = fs.Stat(ix.rootFS, path) + if err == nil { + if fi.Mode().IsRegular() { + ix.readMetadataFile(pi, path) + return + } + } + // no METADATA file exists -- leave nil and finish + } + // Look for the METADATA files to read, and record any missing. + for _, p := range projectsToRead { + go findMeta(p) + } + // Wait until all of the projects have been read. + var msg strings.Builder + result := make([]*ProjectMetadata, 0, len(projects)) + for _, pi := range projectIndexes { + pi.wait() + // Combine any errors into a single error. + if pi.err != nil { + fmt.Fprintf(&msg, " %v\n", pi.err) + } else if pi.pm != nil { + result = append(result, pi.pm) + } + } + if msg.Len() > 0 { + return nil, fmt.Errorf("error reading project(s):\n%s", msg.String()) + } + if len(result) == 0 { + return nil, nil + } + return result, nil +} + +// readMetadataFile tries to read and parse a METADATA file at `path` for `project`. +func (ix *Index) readMetadataFile(pi *projectIndex, path string) { + f, err := ix.rootFS.Open(path) + if err != nil { + pi.err = fmt.Errorf("error opening project %q metadata %q: %w", pi.project, path, err) + return + } + + // read the file + data, err := io.ReadAll(f) + if err != nil { + pi.err = fmt.Errorf("error reading project %q metadata %q: %w", pi.project, path, err) + return + } + f.Close() + + uo := prototext.UnmarshalOptions{DiscardUnknown: true} + pm := &ProjectMetadata{project: pi.project} + err = uo.Unmarshal(data, &pm.proto) + if err != nil { + pi.err = fmt.Errorf("error in project %q metadata %q: %w", pi.project, path, err) + return + } + + pi.pm = pm +} diff --git a/tools/compliance/projectmetadata/projectmetadata_test.go b/tools/compliance/projectmetadata/projectmetadata_test.go new file mode 100644 index 0000000000..1e4256ff5f --- /dev/null +++ b/tools/compliance/projectmetadata/projectmetadata_test.go @@ -0,0 +1,294 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package projectmetadata + +import ( + "fmt" + "strings" + "testing" + + "android/soong/tools/compliance/testfs" +) + +const ( + // EMPTY represents a METADATA file with no recognized fields + EMPTY = `` + + // INVALID_NAME represents a METADATA file with the wrong type of name + INVALID_NAME = `name: a library\n` + + // INVALID_DESCRIPTION represents a METADATA file with the wrong type of description + INVALID_DESCRIPTION = `description: unquoted text\n` + + // INVALID_VERSION represents a METADATA file with the wrong type of version + INVALID_VERSION = `third_party { version: 1 }` + + // MY_LIB_1_0 represents a METADATA file for version 1.0 of mylib + MY_LIB_1_0 = `name: "mylib" description: "my library" third_party { version: "1.0" }` + + // NO_NAME_0_1 represents a METADATA file with a description but no name + NO_NAME_0_1 = `description: "my library" third_party { version: "0.1" }` +) + +func TestReadMetadataForProjects(t *testing.T) { + tests := []struct { + name string + fs *testfs.TestFS + projects []string + expectedError string + expected []pmeta + }{ + { + name: "trivial", + fs: &testfs.TestFS{ + "/a/METADATA": []byte("name: \"Android\"\n"), + }, + projects: []string{"/a"}, + expected: []pmeta{{project: "/a", versionedName: "Android"}}, + }, + { + name: "versioned", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(MY_LIB_1_0), + }, + projects: []string{"/a"}, + expected: []pmeta{{project: "/a", versionedName: "mylib_v_1.0"}}, + }, + { + name: "versioneddesc", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(NO_NAME_0_1), + }, + projects: []string{"/a"}, + expected: []pmeta{{project: "/a", versionedName: "my library"}}, + }, + { + name: "unterminated", + fs: &testfs.TestFS{ + "/a/METADATA": []byte("name: \"Android\n"), + }, + projects: []string{"/a"}, + expectedError: `invalid character '\n' in string`, + }, + { + name: "abc", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(EMPTY), + "/b/METADATA": []byte(MY_LIB_1_0), + "/c/METADATA": []byte(NO_NAME_0_1), + }, + projects: []string{"/a", "/b", "/c"}, + expected: []pmeta{ + {project: "/a", versionedName: ""}, + {project: "/b", versionedName: "mylib_v_1.0"}, + {project: "/c", versionedName: "my library"}, + }, + }, + { + name: "ab", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(EMPTY), + "/b/METADATA": []byte(MY_LIB_1_0), + }, + projects: []string{"/a", "/b", "/c"}, + expected: []pmeta{ + {project: "/a", versionedName: ""}, + {project: "/b", versionedName: "mylib_v_1.0"}, + }, + }, + { + name: "ac", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(EMPTY), + "/c/METADATA": []byte(NO_NAME_0_1), + }, + projects: []string{"/a", "/b", "/c"}, + expected: []pmeta{ + {project: "/a", versionedName: ""}, + {project: "/c", versionedName: "my library"}, + }, + }, + { + name: "bc", + fs: &testfs.TestFS{ + "/b/METADATA": []byte(MY_LIB_1_0), + "/c/METADATA": []byte(NO_NAME_0_1), + }, + projects: []string{"/a", "/b", "/c"}, + expected: []pmeta{ + {project: "/b", versionedName: "mylib_v_1.0"}, + {project: "/c", versionedName: "my library"}, + }, + }, + { + name: "wrongnametype", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_NAME), + }, + projects: []string{"/a"}, + expectedError: `invalid value for string type`, + }, + { + name: "wrongdescriptiontype", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_DESCRIPTION), + }, + projects: []string{"/a"}, + expectedError: `invalid value for string type`, + }, + { + name: "wrongversiontype", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_VERSION), + }, + projects: []string{"/a"}, + expectedError: `invalid value for string type`, + }, + { + name: "wrongtype", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_NAME + INVALID_DESCRIPTION + INVALID_VERSION), + }, + projects: []string{"/a"}, + expectedError: `invalid value for string type`, + }, + { + name: "empty", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(EMPTY), + }, + projects: []string{"/a"}, + expected: []pmeta{{project: "/a", versionedName: ""}}, + }, + { + name: "emptyother", + fs: &testfs.TestFS{ + "/a/METADATA.bp": []byte(EMPTY), + }, + projects: []string{"/a"}, + }, + { + name: "emptyfs", + fs: &testfs.TestFS{}, + projects: []string{"/a"}, + }, + { + name: "override", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_NAME + INVALID_DESCRIPTION + INVALID_VERSION), + "/a/METADATA.android": []byte(MY_LIB_1_0), + }, + projects: []string{"/a"}, + expected: []pmeta{{project: "/a", versionedName: "mylib_v_1.0"}}, + }, + { + name: "enchilada", + fs: &testfs.TestFS{ + "/a/METADATA": []byte(INVALID_NAME + INVALID_DESCRIPTION + INVALID_VERSION), + "/a/METADATA.android": []byte(EMPTY), + "/b/METADATA": []byte(MY_LIB_1_0), + "/c/METADATA": []byte(NO_NAME_0_1), + }, + projects: []string{"/a", "/b", "/c"}, + expected: []pmeta{ + {project: "/a", versionedName: ""}, + {project: "/b", versionedName: "mylib_v_1.0"}, + {project: "/c", versionedName: "my library"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ix := NewIndex(tt.fs) + pms, err := ix.MetadataForProjects(tt.projects...) + if err != nil { + if len(tt.expectedError) == 0 { + t.Errorf("unexpected error: got %s, want no error", err) + } else if !strings.Contains(err.Error(), tt.expectedError) { + t.Errorf("unexpected error: got %s, want %q", err, tt.expectedError) + } + return + } + t.Logf("actual %d project metadata", len(pms)) + for _, pm := range pms { + t.Logf(" %v", pm.String()) + } + t.Logf("expected %d project metadata", len(tt.expected)) + for _, pm := range tt.expected { + t.Logf(" %s", pm.String()) + } + if len(tt.expectedError) > 0 { + t.Errorf("unexpected success: got no error, want %q err", tt.expectedError) + return + } + if len(pms) != len(tt.expected) { + t.Errorf("missing project metadata: got %d project metadata, want %d", len(pms), len(tt.expected)) + } + for i := 0; i < len(pms) && i < len(tt.expected); i++ { + if msg := tt.expected[i].difference(pms[i]); msg != "" { + t.Errorf("unexpected metadata starting at index %d: %s", i, msg) + return + } + } + if len(pms) < len(tt.expected) { + t.Errorf("missing metadata starting at index %d: got nothing, want %s", len(pms), tt.expected[len(pms)].String()) + } + if len(tt.expected) < len(pms) { + t.Errorf("unexpected metadata starting at index %d: got %s, want nothing", len(tt.expected), pms[len(tt.expected)].String()) + } + }) + } +} + +type pmeta struct { + project string + versionedName string +} + +func (pm pmeta) String() string { + return fmt.Sprintf("project: %q versionedName: %q\n", pm.project, pm.versionedName) +} + +func (pm pmeta) equals(other *ProjectMetadata) bool { + if pm.project != other.project { + return false + } + if pm.versionedName != other.VersionedName() { + return false + } + return true +} + +func (pm pmeta) difference(other *ProjectMetadata) string { + if pm.equals(other) { + return "" + } + var sb strings.Builder + fmt.Fprintf(&sb, "got") + if pm.project != other.project { + fmt.Fprintf(&sb, " project: %q", other.project) + } + if pm.versionedName != other.VersionedName() { + fmt.Fprintf(&sb, " versionedName: %q", other.VersionedName()) + } + fmt.Fprintf(&sb, ", want") + if pm.project != other.project { + fmt.Fprintf(&sb, " project: %q", pm.project) + } + if pm.versionedName != other.VersionedName() { + fmt.Fprintf(&sb, " versionedName: %q", pm.versionedName) + } + return sb.String() +} diff --git a/tools/compliance/readgraph.go b/tools/compliance/readgraph.go index 7faca86cde..bf364e66cd 100644 --- a/tools/compliance/readgraph.go +++ b/tools/compliance/readgraph.go @@ -34,10 +34,17 @@ var ( type globalFS struct{} +var _ fs.FS = globalFS{} +var _ fs.StatFS = globalFS{} + func (s globalFS) Open(name string) (fs.File, error) { return os.Open(name) } +func (s globalFS) Stat(name string) (fs.FileInfo, error) { + return os.Stat(name) +} + var FS globalFS // GetFS returns a filesystem for accessing files under the OUT_DIR environment variable. diff --git a/tools/compliance/readgraph_test.go b/tools/compliance/readgraph_test.go index bcf9f39603..a2fb04db4c 100644 --- a/tools/compliance/readgraph_test.go +++ b/tools/compliance/readgraph_test.go @@ -19,12 +19,14 @@ import ( "sort" "strings" "testing" + + "android/soong/tools/compliance/testfs" ) func TestReadLicenseGraph(t *testing.T) { tests := []struct { name string - fs *testFS + fs *testfs.TestFS roots []string expectedError string expectedEdges []edge @@ -32,7 +34,7 @@ func TestReadLicenseGraph(t *testing.T) { }{ { name: "trivial", - fs: &testFS{ + fs: &testfs.TestFS{ "app.meta_lic": []byte("package_name: \"Android\"\n"), }, roots: []string{"app.meta_lic"}, @@ -41,7 +43,7 @@ func TestReadLicenseGraph(t *testing.T) { }, { name: "unterminated", - fs: &testFS{ + fs: &testfs.TestFS{ "app.meta_lic": []byte("package_name: \"Android\n"), }, roots: []string{"app.meta_lic"}, @@ -49,7 +51,7 @@ func TestReadLicenseGraph(t *testing.T) { }, { name: "danglingref", - fs: &testFS{ + fs: &testfs.TestFS{ "app.meta_lic": []byte(AOSP + "deps: {\n file: \"lib.meta_lic\"\n}\n"), }, roots: []string{"app.meta_lic"}, @@ -57,7 +59,7 @@ func TestReadLicenseGraph(t *testing.T) { }, { name: "singleedge", - fs: &testFS{ + fs: &testfs.TestFS{ "app.meta_lic": []byte(AOSP + "deps: {\n file: \"lib.meta_lic\"\n}\n"), "lib.meta_lic": []byte(AOSP), }, @@ -67,7 +69,7 @@ func TestReadLicenseGraph(t *testing.T) { }, { name: "fullgraph", - fs: &testFS{ + fs: &testfs.TestFS{ "apex.meta_lic": []byte(AOSP + "deps: {\n file: \"app.meta_lic\"\n}\ndeps: {\n file: \"bin.meta_lic\"\n}\n"), "app.meta_lic": []byte(AOSP), "bin.meta_lic": []byte(AOSP + "deps: {\n file: \"lib.meta_lic\"\n}\n"), diff --git a/tools/compliance/test_util.go b/tools/compliance/test_util.go index c9d6fe29ec..6c50d3e761 100644 --- a/tools/compliance/test_util.go +++ b/tools/compliance/test_util.go @@ -17,10 +17,11 @@ package compliance import ( "fmt" "io" - "io/fs" "sort" "strings" "testing" + + "android/soong/tools/compliance/testfs" ) const ( @@ -145,51 +146,6 @@ func newTestConditionSet(lg *LicenseGraph, targetName string, conditionName []st return cs } -// testFS implements a test file system (fs.FS) simulated by a map from filename to []byte content. -type testFS map[string][]byte - -// Open implements fs.FS.Open() to open a file based on the filename. -func (fs *testFS) Open(name string) (fs.File, error) { - if _, ok := (*fs)[name]; !ok { - return nil, fmt.Errorf("unknown file %q", name) - } - return &testFile{fs, name, 0}, nil -} - -// testFile implements a test file (fs.File) based on testFS above. -type testFile struct { - fs *testFS - name string - posn int -} - -// Stat not implemented to obviate implementing fs.FileInfo. -func (f *testFile) Stat() (fs.FileInfo, error) { - return nil, fmt.Errorf("unimplemented") -} - -// Read copies bytes from the testFS map. -func (f *testFile) Read(b []byte) (int, error) { - if f.posn < 0 { - return 0, fmt.Errorf("file not open: %q", f.name) - } - if f.posn >= len((*f.fs)[f.name]) { - return 0, io.EOF - } - n := copy(b, (*f.fs)[f.name][f.posn:]) - f.posn += n - return n, nil -} - -// Close marks the testFile as no longer in use. -func (f *testFile) Close() error { - if f.posn < 0 { - return fmt.Errorf("file already closed: %q", f.name) - } - f.posn = -1 - return nil -} - // edge describes test data edges to define test graphs. type edge struct { target, dep string @@ -268,7 +224,7 @@ func toGraph(stderr io.Writer, roots []string, edges []annotated) (*LicenseGraph deps[edge.dep] = []annotated{} } } - fs := make(testFS) + fs := make(testfs.TestFS) for file, edges := range deps { body := meta[file] for _, edge := range edges { diff --git a/tools/compliance/testfs/Android.bp b/tools/compliance/testfs/Android.bp new file mode 100644 index 0000000000..6baaf18b66 --- /dev/null +++ b/tools/compliance/testfs/Android.bp @@ -0,0 +1,25 @@ +// Copyright (C) 2022 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +bootstrap_go_package { + name: "compliance-test-fs-module", + srcs: [ + "testfs.go", + ], + pkgPath: "android/soong/tools/compliance/testfs", +} diff --git a/tools/compliance/testfs/testfs.go b/tools/compliance/testfs/testfs.go new file mode 100644 index 0000000000..2c75c5b1d7 --- /dev/null +++ b/tools/compliance/testfs/testfs.go @@ -0,0 +1,129 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfs + +import ( + "fmt" + "io" + "io/fs" + "strings" + "time" +) + +// TestFS implements a test file system (fs.FS) simulated by a map from filename to []byte content. +type TestFS map[string][]byte + +var _ fs.FS = (*TestFS)(nil) +var _ fs.StatFS = (*TestFS)(nil) + +// Open implements fs.FS.Open() to open a file based on the filename. +func (tfs *TestFS) Open(name string) (fs.File, error) { + if _, ok := (*tfs)[name]; !ok { + return nil, fmt.Errorf("unknown file %q", name) + } + return &TestFile{tfs, name, 0}, nil +} + +// Stat implements fs.StatFS.Stat() to examine a file based on the filename. +func (tfs *TestFS) Stat(name string) (fs.FileInfo, error) { + if content, ok := (*tfs)[name]; ok { + return &TestFileInfo{name, len(content), 0666}, nil + } + dirname := name + if !strings.HasSuffix(dirname, "/") { + dirname = dirname + "/" + } + for name := range (*tfs) { + if strings.HasPrefix(name, dirname) { + return &TestFileInfo{name, 8, fs.ModeDir | fs.ModePerm}, nil + } + } + return nil, fmt.Errorf("file not found: %q", name) +} + +// TestFileInfo implements a file info (fs.FileInfo) based on TestFS above. +type TestFileInfo struct { + name string + size int + mode fs.FileMode +} + +var _ fs.FileInfo = (*TestFileInfo)(nil) + +// Name returns the name of the file +func (fi *TestFileInfo) Name() string { + return fi.name +} + +// Size returns the size of the file in bytes. +func (fi *TestFileInfo) Size() int64 { + return int64(fi.size) +} + +// Mode returns the fs.FileMode bits. +func (fi *TestFileInfo) Mode() fs.FileMode { + return fi.mode +} + +// ModTime fakes a modification time. +func (fi *TestFileInfo) ModTime() time.Time { + return time.UnixMicro(0xb0bb) +} + +// IsDir is a synonym for Mode().IsDir() +func (fi *TestFileInfo) IsDir() bool { + return fi.mode.IsDir() +} + +// Sys is unused and returns nil. +func (fi *TestFileInfo) Sys() any { + return nil +} + +// TestFile implements a test file (fs.File) based on TestFS above. +type TestFile struct { + fs *TestFS + name string + posn int +} + +var _ fs.File = (*TestFile)(nil) + +// Stat not implemented to obviate implementing fs.FileInfo. +func (f *TestFile) Stat() (fs.FileInfo, error) { + return f.fs.Stat(f.name) +} + +// Read copies bytes from the TestFS map. +func (f *TestFile) Read(b []byte) (int, error) { + if f.posn < 0 { + return 0, fmt.Errorf("file not open: %q", f.name) + } + if f.posn >= len((*f.fs)[f.name]) { + return 0, io.EOF + } + n := copy(b, (*f.fs)[f.name][f.posn:]) + f.posn += n + return n, nil +} + +// Close marks the TestFile as no longer in use. +func (f *TestFile) Close() error { + if f.posn < 0 { + return fmt.Errorf("file already closed: %q", f.name) + } + f.posn = -1 + return nil +} |