From ab3e717feaf76784e3939592d98d788d19a4c022 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Fri, 7 Oct 2022 15:39:20 -0400 Subject: chore: add idsercher functions for 2.3 Signed-off-by: Keith Zantow --- idsearcher/idsearcher.go | 125 ++++++++++++++++++ idsearcher/idsearcher_test.go | 289 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+) diff --git a/idsearcher/idsearcher.go b/idsearcher/idsearcher.go index 29b0faa..a5176ca 100644 --- a/idsearcher/idsearcher.go +++ b/idsearcher/idsearcher.go @@ -7,6 +7,7 @@ package idsearcher import ( "bufio" "fmt" + "github.com/spdx/tools-golang/spdx/v2_3" "os" "path/filepath" "regexp" @@ -267,6 +268,130 @@ func BuildIDsDocument2_2(packageName string, dirRoot string, idconfig *Config2_2 return doc, nil } +// ===== 2.3 Searcher functions ===== + +// Config2_3 is a collection of configuration settings for docbuilder +// (for version 2.3 SPDX Documents). A few mandatory fields are set here +// so that they can be repeatedly reused in multiple calls to Build2_3. +type Config2_3 struct { + // NamespacePrefix should be a URI representing a prefix for the + // namespace with which the SPDX Document will be associated. + // It will be used in the DocumentNamespace field in the CreationInfo + // section, followed by the per-Document package name and a random UUID. + NamespacePrefix string + + // BuilderPathsIgnored lists certain paths to be omitted from the built + // document. Each string should be a path, relative to the package's + // dirRoot, to a specific file or (for all files in a directory) ending + // in a slash. Prefix the string with "**" to omit all instances of that + // file / directory, regardless of where it is in the file tree. + BuilderPathsIgnored []string + + // SearcherPathsIgnored lists certain paths that should not be searched + // by idsearcher, even if those paths have Files present. It uses the + // same format as BuilderPathsIgnored. + SearcherPathsIgnored []string +} + +// BuildIDsDocument2_3 creates an SPDX Document (version 2.3) and searches for +// short-form IDs in each file, filling in license fields as appropriate. It +// returns that document or error if any is encountered. Arguments: +// - packageName: name of package / directory +// - dirRoot: path to directory to be analyzed +// - namespacePrefix: URI representing a prefix for the +// namespace with which the SPDX Document will be associated +func BuildIDsDocument2_3(packageName string, dirRoot string, idconfig *Config2_3) (*v2_3.Document, error) { + // first, build the Document using builder + bconfig := &builder.Config2_3{ + NamespacePrefix: idconfig.NamespacePrefix, + CreatorType: "Tool", + Creator: "github.com/spdx/tools-golang/idsearcher", + PathsIgnored: idconfig.BuilderPathsIgnored, + } + doc, err := builder.Build2_3(packageName, dirRoot, bconfig) + if err != nil { + return nil, err + } + if doc == nil { + return nil, fmt.Errorf("builder returned nil Document") + } + if doc.Packages == nil { + return nil, fmt.Errorf("builder returned nil Packages map") + } + if len(doc.Packages) != 1 { + return nil, fmt.Errorf("builder returned %d Packages", len(doc.Packages)) + } + + // now, walk through each file and find its licenses (if any) + pkg := doc.Packages[0] + if pkg == nil { + return nil, fmt.Errorf("builder returned nil Package") + } + if pkg.Files == nil { + return nil, fmt.Errorf("builder returned nil Files in Package") + } + licsForPackage := map[string]int{} + for _, f := range pkg.Files { + // start by initializing / clearing values + f.LicenseInfoInFiles = []string{"NOASSERTION"} + f.LicenseConcluded = "NOASSERTION" + + // check whether the searcher should ignore this file + if utils.ShouldIgnore(f.FileName, idconfig.SearcherPathsIgnored) { + continue + } + + fPath := filepath.Join(dirRoot, f.FileName) + // FIXME this is not preferable -- ignoring error + ids, _ := searchFileIDs(fPath) + // FIXME for now, proceed onwards with whatever IDs we obtained. + // FIXME instead of ignoring the error, should probably either log it, + // FIXME and/or enable the caller to configure what should happen. + + // separate out for this file's licenses + licsForFile := map[string]int{} + licsParens := []string{} + for _, lid := range ids { + // get individual elements and add for file and package + licElements := getIndividualLicenses(lid) + for _, elt := range licElements { + licsForFile[elt] = 1 + licsForPackage[elt] = 1 + } + // parenthesize if needed and add to slice for joining + licsParens = append(licsParens, makeElement(lid)) + } + + // OK -- now we can fill in the file's details, or NOASSERTION if none + if len(licsForFile) > 0 { + f.LicenseInfoInFiles = []string{} + for lic := range licsForFile { + f.LicenseInfoInFiles = append(f.LicenseInfoInFiles, lic) + } + sort.Strings(f.LicenseInfoInFiles) + // avoid adding parens and joining for single-ID items + if len(licsParens) == 1 { + f.LicenseConcluded = ids[0] + } else { + f.LicenseConcluded = strings.Join(licsParens, " AND ") + } + } + } + + // and finally, we can fill in the package's details + if len(licsForPackage) == 0 { + pkg.PackageLicenseInfoFromFiles = []string{"NOASSERTION"} + } else { + pkg.PackageLicenseInfoFromFiles = []string{} + for lic := range licsForPackage { + pkg.PackageLicenseInfoFromFiles = append(pkg.PackageLicenseInfoFromFiles, lic) + } + sort.Strings(pkg.PackageLicenseInfoFromFiles) + } + + return doc, nil +} + // ===== Utility functions (not version-specific) ===== func searchFileIDs(filePath string) ([]string, error) { idsMap := map[string]int{} diff --git a/idsearcher/idsearcher_test.go b/idsearcher/idsearcher_test.go index 00e5206..7d7a5bb 100644 --- a/idsearcher/idsearcher_test.go +++ b/idsearcher/idsearcher_test.go @@ -584,6 +584,295 @@ func Test2_2SearcherFailsWithInvalidPath(t *testing.T) { } } +// ===== 2.3 Searcher top-level function tests ===== +func Test2_3SearcherCanFillInIDs(t *testing.T) { + packageName := "project2" + dirRoot := "../testdata/project2/" + config := &Config2_3{ + NamespacePrefix: "https://github.com/swinslow/spdx-docs/spdx-go/testdata-", + } + + doc, err := BuildIDsDocument2_3(packageName, dirRoot, config) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if doc == nil { + t.Fatalf("expected non-nil Document, got nil") + } + + // not checking all contents of doc, see builder tests for those + + // get the package and its files, checking size of each + if doc.Packages == nil { + t.Fatalf("expected non-nil Packages, got nil") + } + if len(doc.Packages) != 1 { + t.Fatalf("expected Packages len to be 1, got %d", len(doc.Packages)) + } + pkg := doc.Packages[0] + if pkg == nil { + t.Fatalf("expected non-nil pkg, got nil") + } + + if pkg.Files == nil { + t.Fatalf("expected non-nil Files, got nil") + } + if len(pkg.Files) != 6 { + t.Fatalf("expected Files len to be 6, got %d", len(pkg.Files)) + } + + fileInFolder := pkg.Files[0] + if fileInFolder.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileInFolder.LicenseInfoInFiles) != 1 { + t.Fatalf("expected LicenseInfoInFiles len to be 1, got %d", len(fileInFolder.LicenseInfoInFiles)) + } + if fileInFolder.LicenseInfoInFiles[0] != "MIT" { + t.Errorf("expected %v, got %v", "MIT", fileInFolder.LicenseInfoInFiles[0]) + } + if fileInFolder.LicenseConcluded != "MIT" { + t.Errorf("expected %v, got %v", "MIT", fileInFolder.LicenseConcluded) + } + + fileTrailingComment := pkg.Files[1] + if fileTrailingComment.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileTrailingComment.LicenseInfoInFiles) != 1 { + t.Fatalf("expected LicenseInfoInFiles len to be 1, got %d", len(fileTrailingComment.LicenseInfoInFiles)) + } + if fileTrailingComment.LicenseInfoInFiles[0] != "GPL-2.0-or-later" { + t.Errorf("expected %v, got %v", "GPL-2.0-or-later", fileTrailingComment.LicenseInfoInFiles[0]) + } + if fileTrailingComment.LicenseConcluded != "GPL-2.0-or-later" { + t.Errorf("expected %v, got %v", "GPL-2.0-or-later", fileTrailingComment.LicenseConcluded) + } + + fileHasDuplicateID := pkg.Files[2] + if fileHasDuplicateID.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileHasDuplicateID.LicenseInfoInFiles) != 1 { + t.Fatalf("expected LicenseInfoInFiles len to be 1, got %d", len(fileHasDuplicateID.LicenseInfoInFiles)) + } + if fileHasDuplicateID.LicenseInfoInFiles[0] != "MIT" { + t.Errorf("expected %v, got %v", "MIT", fileHasDuplicateID.LicenseInfoInFiles[0]) + } + if fileHasDuplicateID.LicenseConcluded != "MIT" { + t.Errorf("expected %v, got %v", "MIT", fileHasDuplicateID.LicenseConcluded) + } + + fileHasID := pkg.Files[3] + if fileHasID.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileHasID.LicenseInfoInFiles) != 2 { + t.Fatalf("expected LicenseInfoInFiles len to be 2, got %d", len(fileHasID.LicenseInfoInFiles)) + } + if fileHasID.LicenseInfoInFiles[0] != "Apache-2.0" { + t.Errorf("expected %v, got %v", "Apache-2.0", fileHasID.LicenseInfoInFiles[0]) + } + if fileHasID.LicenseInfoInFiles[1] != "GPL-2.0-or-later" { + t.Errorf("expected %v, got %v", "GPL-2.0-or-later", fileHasID.LicenseInfoInFiles[1]) + } + if fileHasID.LicenseConcluded != "Apache-2.0 OR GPL-2.0-or-later" { + t.Errorf("expected %v, got %v", "Apache-2.0 OR GPL-2.0-or-later", fileHasID.LicenseConcluded) + } + + fileMultipleIDs := pkg.Files[4] + if fileMultipleIDs.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileMultipleIDs.LicenseInfoInFiles) != 5 { + t.Fatalf("expected LicenseInfoInFiles len to be 5, got %d", len(fileMultipleIDs.LicenseInfoInFiles)) + } + if fileMultipleIDs.LicenseInfoInFiles[0] != "BSD-2-Clause" { + t.Errorf("expected %v, got %v", "BSD-2-Clause", fileMultipleIDs.LicenseInfoInFiles[0]) + } + if fileMultipleIDs.LicenseInfoInFiles[1] != "BSD-3-Clause" { + t.Errorf("expected %v, got %v", "BSD-3-Clause", fileMultipleIDs.LicenseInfoInFiles[1]) + } + // here, DO NOT keep the + + if fileMultipleIDs.LicenseInfoInFiles[2] != "EPL-1.0" { + t.Errorf("expected %v, got %v", "EPL-1.0", fileMultipleIDs.LicenseInfoInFiles[2]) + } + if fileMultipleIDs.LicenseInfoInFiles[3] != "ISC" { + t.Errorf("expected %v, got %v", "ISC", fileMultipleIDs.LicenseInfoInFiles[3]) + } + if fileMultipleIDs.LicenseInfoInFiles[4] != "MIT" { + t.Errorf("expected %v, got %v", "MIT", fileMultipleIDs.LicenseInfoInFiles[4]) + } + if fileMultipleIDs.LicenseConcluded != "((MIT AND BSD-3-Clause) OR ISC) AND BSD-2-Clause AND EPL-1.0+" { + t.Errorf("expected %v, got %v", "((MIT AND BSD-3-Clause) OR ISC) AND BSD-2-Clause AND EPL-1.0+", fileMultipleIDs.LicenseConcluded) + } + + fileNoID := pkg.Files[5] + if fileNoID.LicenseInfoInFiles == nil { + t.Fatalf("expected non-nil LicenseInfoInFiles, got nil") + } + if len(fileNoID.LicenseInfoInFiles) != 1 { + t.Fatalf("expected LicenseInfoInFiles len to be 1, got %d", len(fileNoID.LicenseInfoInFiles)) + } + if fileNoID.LicenseInfoInFiles[0] != "NOASSERTION" { + t.Errorf("expected %v, got %v", "NOASSERTION", fileNoID.LicenseInfoInFiles[0]) + } + if fileNoID.LicenseConcluded != "NOASSERTION" { + t.Errorf("expected %v, got %v", "NOASSERTION", fileNoID.LicenseConcluded) + } + + // and finally, the package should have all of these licenses + if pkg.PackageLicenseInfoFromFiles == nil { + t.Fatalf("expected non-nil PackageLicenseInfoFromFiles, got nil") + } + if len(pkg.PackageLicenseInfoFromFiles) != 7 { + t.Fatalf("expected PackageLicenseInfoFromFiles len to be 7, got %d", len(pkg.PackageLicenseInfoFromFiles)) + } + if pkg.PackageLicenseInfoFromFiles[0] != "Apache-2.0" { + t.Errorf("expected %v, got %v", "Apache-2.0", pkg.PackageLicenseInfoFromFiles[0]) + } + if pkg.PackageLicenseInfoFromFiles[1] != "BSD-2-Clause" { + t.Errorf("expected %v, got %v", "BSD-2-Clause", pkg.PackageLicenseInfoFromFiles[1]) + } + if pkg.PackageLicenseInfoFromFiles[2] != "BSD-3-Clause" { + t.Errorf("expected %v, got %v", "BSD-3-Clause", pkg.PackageLicenseInfoFromFiles[2]) + } + // here, DO NOT keep the + + if pkg.PackageLicenseInfoFromFiles[3] != "EPL-1.0" { + t.Errorf("expected %v, got %v", "EPL-1.0", pkg.PackageLicenseInfoFromFiles[3]) + } + if pkg.PackageLicenseInfoFromFiles[4] != "GPL-2.0-or-later" { + t.Errorf("expected %v, got %v", "GPL-2.0-or-later", pkg.PackageLicenseInfoFromFiles[4]) + } + if pkg.PackageLicenseInfoFromFiles[5] != "ISC" { + t.Errorf("expected %v, got %v", "ISC", pkg.PackageLicenseInfoFromFiles[5]) + } + if pkg.PackageLicenseInfoFromFiles[6] != "MIT" { + t.Errorf("expected %v, got %v", "MIT", pkg.PackageLicenseInfoFromFiles[6]) + } + +} + +func Test2_3SearcherCanFillInIDsAndIgnorePaths(t *testing.T) { + packageName := "project3" + dirRoot := "../testdata/project3/" + config := &Config2_3{ + NamespacePrefix: "https://github.com/swinslow/spdx-docs/spdx-go/testdata-", + BuilderPathsIgnored: []string{ + "**/ignoredir/", + "/excludedir/", + "**/ignorefile.txt", + "/alsoEXCLUDEthis.txt", + }, + SearcherPathsIgnored: []string{ + "**/dontscan.txt", + }, + } + + doc, err := BuildIDsDocument2_3(packageName, dirRoot, config) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if doc == nil { + t.Fatalf("expected non-nil Document, got nil") + } + + // not checking all contents of doc, see builder tests for those + + // get the package and its files, checking licenses for each, and + // confirming NOASSERTION for those that are skipped + pkg := doc.Packages[0] + if pkg == nil { + t.Fatalf("expected non-nil pkg, got nil") + } + if len(pkg.Files) != 5 { + t.Fatalf("expected len %d, got %d", 5, len(pkg.Files)) + } + + f := pkg.Files[0] + if f.FileName != "./dontscan.txt" { + t.Errorf("expected %v, got %v", "./dontscan.txt", f.FileName) + } + if len(f.LicenseInfoInFiles) != 1 { + t.Errorf("expected len to be %d, got %d", 1, len(f.LicenseInfoInFiles)) + } + if f.LicenseInfoInFiles[0] != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseInfoInFiles[0]) + } + if f.LicenseConcluded != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseConcluded) + } + + f = pkg.Files[1] + if f.FileName != "./keep/keep.txt" { + t.Errorf("expected %v, got %v", "./keep/keep.txt", f.FileName) + } + if len(f.LicenseInfoInFiles) != 1 { + t.Errorf("expected len to be %d, got %d", 1, len(f.LicenseInfoInFiles)) + } + if f.LicenseInfoInFiles[0] != "MIT" { + t.Errorf("expected %s, got %s", "MIT", f.LicenseInfoInFiles[0]) + } + if f.LicenseConcluded != "MIT" { + t.Errorf("expected %s, got %s", "MIT", f.LicenseConcluded) + } + + f = pkg.Files[2] + if f.FileName != "./keep.txt" { + t.Errorf("expected %v, got %v", "./keep.txt", f.FileName) + } + if len(f.LicenseInfoInFiles) != 1 { + t.Errorf("expected len to be %d, got %d", 1, len(f.LicenseInfoInFiles)) + } + if f.LicenseInfoInFiles[0] != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseInfoInFiles[0]) + } + if f.LicenseConcluded != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseConcluded) + } + + f = pkg.Files[3] + if f.FileName != "./subdir/keep/dontscan.txt" { + t.Errorf("expected %v, got %v", "./subdir/keep/dontscan.txt", f.FileName) + } + if len(f.LicenseInfoInFiles) != 1 { + t.Errorf("expected len to be %d, got %d", 1, len(f.LicenseInfoInFiles)) + } + if f.LicenseInfoInFiles[0] != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseInfoInFiles[0]) + } + if f.LicenseConcluded != "NOASSERTION" { + t.Errorf("expected %s, got %s", "NOASSERTION", f.LicenseConcluded) + } + + f = pkg.Files[4] + if f.FileName != "./subdir/keep/keep.txt" { + t.Errorf("expected %v, got %v", "./subdir/keep/keep.txt", f.FileName) + } + if len(f.LicenseInfoInFiles) != 1 { + t.Errorf("expected len to be %d, got %d", 1, len(f.LicenseInfoInFiles)) + } + if f.LicenseInfoInFiles[0] != "MIT" { + t.Errorf("expected %s, got %s", "MIT", f.LicenseInfoInFiles[0]) + } + if f.LicenseConcluded != "MIT" { + t.Errorf("expected %s, got %s", "MIT", f.LicenseConcluded) + } +} + +func Test2_3SearcherFailsWithInvalidPath(t *testing.T) { + packageName := "project2" + dirRoot := "./oops/invalid" + config := &Config2_3{ + NamespacePrefix: "whatever", + } + + _, err := BuildIDsDocument2_3(packageName, dirRoot, config) + if err == nil { + t.Fatalf("expected non-nil error, got nil") + } +} + // ===== Searcher utility tests ===== func TestCanFindShortFormIDWhenPresent(t *testing.T) { filePath := "../testdata/project2/has-id.txt" -- cgit v1.2.3