internal/gopathwalk: create

Extract goimports' logic for walking Go source directories into a
separate package, suitable for use in go/packages. No functional
changes. Added a convenience feature to fastwalk, allowing the user to
say that they're done with a directory and stop receiving callbacks for
it.

Testing is a little light; I expect goimports' tests to cover most
everything we care about.

Change-Id: If047ada4414f5f282637d11fd07e8342fadc9c33
Reviewed-on: https://go-review.googlesource.com/c/138877
Run-TryBot: Heschi Kreinick <heschi@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>

2 files changed, 10 insertions, 253 deletions

diff --git a/imports/fix.go b/imports/fix.go
index d9cbe656e..75d37f894 100644
--- a/imports/fix.go
+++ b/imports/fix.go
@@ -5,8 +5,6 @@
 package imports
 
 import (
-	"bufio"
-	"bytes"
 	"context"
 	"fmt"
 	"go/ast"
@@ -24,7 +22,7 @@ import (
 	"sync"
 
 	"golang.org/x/tools/go/ast/astutil"
-	"golang.org/x/tools/internal/fastwalk"
+	"golang.org/x/tools/internal/gopathwalk"
 )
 
 // Debug controls verbose logging.
@@ -523,195 +521,27 @@ func distance(basepath, targetpath string) int {
 	return strings.Count(p, string(filepath.Separator)) + 1
 }
 
-// getIgnoredDirs reads an optional config file at <path>/.goimportsignore
-// of relative directories to ignore when scanning for go files.
-// The provided path is one of the $GOPATH entries with "src" appended.
-func getIgnoredDirs(path string) []os.FileInfo {
-	file := filepath.Join(path, ".goimportsignore")
-	slurp, err := ioutil.ReadFile(file)
-	if Debug {
-		if err != nil {
-			log.Print(err)
-		} else {
-			log.Printf("Read %s", file)
-		}
-	}
-	if err != nil {
-		return nil
-	}
-
-	var ignoredDirs []os.FileInfo
-	bs := bufio.NewScanner(bytes.NewReader(slurp))
-	for bs.Scan() {
-		line := strings.TrimSpace(bs.Text())
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		full := filepath.Join(path, line)
-		if fi, err := os.Stat(full); err == nil {
-			ignoredDirs = append(ignoredDirs, fi)
-			if Debug {
-				log.Printf("Directory added to ignore list: %s", full)
-			}
-		} else if Debug {
-			log.Printf("Error statting entry in .goimportsignore: %v", err)
-		}
-	}
-	return ignoredDirs
-}
-
-func shouldSkipDir(fi os.FileInfo, ignoredDirs []os.FileInfo) bool {
-	for _, ignoredDir := range ignoredDirs {
-		if os.SameFile(fi, ignoredDir) {
-			return true
-		}
-	}
-	return false
-}
-
-// shouldTraverse reports whether the symlink fi, found in dir,
-// should be followed.  It makes sure symlinks were never visited
-// before to avoid symlink loops.
-func shouldTraverse(dir string, fi os.FileInfo, ignoredDirs []os.FileInfo) bool {
-	path := filepath.Join(dir, fi.Name())
-	target, err := filepath.EvalSymlinks(path)
-	if err != nil {
-		return false
-	}
-	ts, err := os.Stat(target)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		return false
-	}
-	if !ts.IsDir() {
-		return false
-	}
-	if shouldSkipDir(ts, ignoredDirs) {
-		return false
-	}
-	// Check for symlink loops by statting each directory component
-	// and seeing if any are the same file as ts.
-	for {
-		parent := filepath.Dir(path)
-		if parent == path {
-			// Made it to the root without seeing a cycle.
-			// Use this symlink.
-			return true
-		}
-		parentInfo, err := os.Stat(parent)
-		if err != nil {
-			return false
-		}
-		if os.SameFile(ts, parentInfo) {
-			// Cycle. Don't traverse.
-			return false
-		}
-		path = parent
-	}
-
-}
-
-var testHookScanDir = func(dir string) {}
-
 // scanGoDirs populates the dirScan map for GOPATH and GOROOT.
 func scanGoDirs() map[string]*pkg {
 	result := make(map[string]*pkg)
 	var mu sync.Mutex
 
-	for _, srcDir := range build.Default.SrcDirs() {
-		if Debug {
-			log.Printf("scanning %s", srcDir)
-		}
-
-		testHookScanDir(srcDir)
-		w := &walker{
-			srcDir:      srcDir,
-			srcV:        filepath.Join(srcDir, "v"),
-			srcMod:      filepath.Join(srcDir, "mod"),
-			ignoredDirs: getIgnoredDirs(srcDir),
-			dirScanMu:   &mu,
-			dirScan:     result,
-		}
-		if err := fastwalk.Walk(srcDir, w.walk); err != nil {
-			log.Printf("goimports: scanning directory %v: %v", srcDir, err)
-		}
-
-		if Debug {
-			defer log.Printf("scanned %s", srcDir)
-		}
-	}
-	return result
-}
-
-// walker is the callback for fastwalk.Walk.
-type walker struct {
-	srcDir       string        // The source directory to scan.
-	srcV, srcMod string        // vgo-style module cache dirs. Optional.
-	ignoredDirs  []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
-
-	dirScanMu *sync.Mutex     // The shared mutex guarding dirScan.
-	dirScan   map[string]*pkg // The results of the scan, sharable across multiple Walk calls.
-}
+	add := func(srcDir, dir string) {
+		mu.Lock()
+		defer mu.Unlock()
 
-func (w *walker) walk(path string, typ os.FileMode) error {
-	if path == w.srcV || path == w.srcMod {
-		return filepath.SkipDir
-	}
-	dir := filepath.Dir(path)
-	if typ.IsRegular() {
-		if dir == w.srcDir {
-			// Doesn't make sense to have regular files
-			// directly in your $GOPATH/src or $GOROOT/src.
-			return nil
-		}
-		if !strings.HasSuffix(path, ".go") {
-			return nil
-		}
-
-		w.dirScanMu.Lock()
-		defer w.dirScanMu.Unlock()
-		if _, dup := w.dirScan[dir]; dup {
-			return nil
+		if _, dup := result[dir]; dup {
+			return
 		}
-		importpath := filepath.ToSlash(dir[len(w.srcDir)+len("/"):])
-		w.dirScan[dir] = &pkg{
+		importpath := filepath.ToSlash(dir[len(srcDir)+len("/"):])
+		result[dir] = &pkg{
 			importPath:      importpath,
 			importPathShort: VendorlessPath(importpath),
 			dir:             dir,
 		}
-		return nil
 	}
-	if typ == os.ModeDir {
-		base := filepath.Base(path)
-		if base == "" || base[0] == '.' || base[0] == '_' ||
-			base == "testdata" || base == "node_modules" {
-			return filepath.SkipDir
-		}
-		fi, err := os.Lstat(path)
-		if err == nil && shouldSkipDir(fi, w.ignoredDirs) {
-			if Debug {
-				log.Printf("skipping directory %q under %s", fi.Name(), dir)
-			}
-			return filepath.SkipDir
-		}
-		return nil
-	}
-	if typ == os.ModeSymlink {
-		base := filepath.Base(path)
-		if strings.HasPrefix(base, ".#") {
-			// Emacs noise.
-			return nil
-		}
-		fi, err := os.Lstat(path)
-		if err != nil {
-			// Just ignore it.
-			return nil
-		}
-		if shouldTraverse(dir, fi, w.ignoredDirs) {
-			return fastwalk.TraverseLink
-		}
-	}
-	return nil
+	gopathwalk.Walk(add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false})
+	return result
 }
 
 // VendorlessPath returns the devendorized version of the import path ipath.
diff --git a/imports/fix_test.go b/imports/fix_test.go
index 4a4315050..f78fb5dff 100644
--- a/imports/fix_test.go
+++ b/imports/fix_test.go
@@ -1559,10 +1559,8 @@ func withEmptyGoPath(fn func()) {
 	oldCompiler := build.Default.Compiler
 	build.Default.GOPATH = ""
 	build.Default.Compiler = "gc"
-	testHookScanDir = func(string) {}
 
 	defer func() {
-		testHookScanDir = func(string) {}
 		build.Default.GOPATH = oldGOPATH
 		build.Default.GOROOT = oldGOROOT
 		build.Default.Compiler = oldCompiler
@@ -2268,77 +2266,6 @@ func TestPkgIsCandidate(t *testing.T) {
 	}
 }
 
-func TestShouldTraverse(t *testing.T) {
-	switch runtime.GOOS {
-	case "windows", "plan9":
-		t.Skipf("skipping symlink-requiring test on %s", runtime.GOOS)
-	}
-
-	dir, err := ioutil.TempDir("", "goimports-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	// Note: mapToDir prepends "src" to each element, since
-	// mapToDir was made for creating GOPATHs.
-	if err := mapToDir(dir, map[string]string{
-		"foo/foo2/file.txt":        "",
-		"foo/foo2/link-to-src":     "LINK:" + dir + "/src",
-		"foo/foo2/link-to-src-foo": "LINK:" + dir + "/src/foo",
-		"foo/foo2/link-to-dot":     "LINK:.",
-		"bar/bar2/file.txt":        "",
-		"bar/bar2/link-to-src-foo": "LINK:" + dir + "/src/foo",
-
-		"a/b/c": "LINK:" + dir + "/src/a/d",
-		"a/d/e": "LINK:" + dir + "/src/a/b",
-	}); err != nil {
-		t.Fatal(err)
-	}
-	tests := []struct {
-		dir  string
-		file string
-		want bool
-	}{
-		{
-			dir:  dir + "/src/foo/foo2",
-			file: "link-to-src-foo",
-			want: false, // loop
-		},
-		{
-			dir:  dir + "/src/foo/foo2",
-			file: "link-to-src",
-			want: false, // loop
-		},
-		{
-			dir:  dir + "/src/foo/foo2",
-			file: "link-to-dot",
-			want: false, // loop
-		},
-		{
-			dir:  dir + "/src/bar/bar2",
-			file: "link-to-src-foo",
-			want: true, // not a loop
-		},
-		{
-			dir:  dir + "/src/a/b/c",
-			file: "e",
-			want: false, // loop: "e" is the same as "b".
-		},
-	}
-	for i, tt := range tests {
-		fi, err := os.Stat(filepath.Join(tt.dir, tt.file))
-		if err != nil {
-			t.Errorf("%d. Stat = %v", i, err)
-			continue
-		}
-		got := shouldTraverse(tt.dir, fi, nil)
-		if got != tt.want {
-			t.Errorf("%d. shouldTraverse(%q, %q) = %v; want %v", i, tt.dir, tt.file, got, tt.want)
-		}
-	}
-}
-
 // Issue 20941: this used to panic on Windows.
 func TestProcessStdin(t *testing.T) {
 	got, err := Process("<standard input>", []byte("package main\nfunc main() {\n\tfmt.Println(123)\n}\n"), nil)