10 files changed, 538 insertions, 0 deletions
diff --git a/internal/robustio/copyfiles.go b/internal/robustio/copyfiles.go
new file mode 100644
index 000000000..6e9f4b387
--- /dev/null
+++ b/internal/robustio/copyfiles.go
@@ -0,0 +1,117 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build ignore
+// +build ignore
+
+// The copyfiles script copies the contents of the internal cmd/go robustio
+// package to the current directory, with adjustments to make it build.
+//
+// NOTE: In retrospect this script got out of hand, as we have to perform
+// various operations on the package to get it to build at old Go versions. If
+// in the future it proves to be flaky, delete it and just copy code manually.
+package main
+
+import (
+	"bytes"
+	"go/build/constraint"
+	"go/scanner"
+	"go/token"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+func main() {
+	dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "go", "internal", "robustio")
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		log.Fatalf("reading the robustio dir: %v", err)
+	}
+
+	// Collect file content so that we can validate before copying.
+	fileContent := make(map[string][]byte)
+	windowsImport := []byte("\t\"internal/syscall/windows\"\n")
+	foundWindowsImport := false
+	for _, entry := range entries {
+		if strings.HasSuffix(entry.Name(), ".go") {
+			pth := filepath.Join(dir, entry.Name())
+			content, err := os.ReadFile(pth)
+			if err != nil {
+				log.Fatalf("reading %q: %v", entry.Name(), err)
+			}
+
+			// Replace the use of internal/syscall/windows.ERROR_SHARING_VIOLATION
+			// with a local constant.
+			if entry.Name() == "robustio_windows.go" && bytes.Contains(content, windowsImport) {
+				foundWindowsImport = true
+				content = bytes.Replace(content, windowsImport, nil, 1)
+				content = bytes.Replace(content, []byte("windows.ERROR_SHARING_VIOLATION"), []byte("ERROR_SHARING_VIOLATION"), -1)
+			}
+
+			// Replace os.ReadFile with ioutil.ReadFile (for 1.15 and older). We
+			// attempt to match calls (via the '('), to avoid matching mentions of
+			// os.ReadFile in comments.
+			//
+			// TODO(rfindley): once we (shortly!) no longer support 1.15, remove
+			// this and break the build.
+			if bytes.Contains(content, []byte("os.ReadFile(")) {
+				content = bytes.Replace(content, []byte("\"os\""), []byte("\"io/ioutil\"\n\t\"os\""), 1)
+				content = bytes.Replace(content, []byte("os.ReadFile("), []byte("ioutil.ReadFile("), -1)
+			}
+
+			// Add +build constraints, for 1.16.
+			content = addPlusBuildConstraints(content)
+
+			fileContent[entry.Name()] = content
+		}
+	}
+
+	if !foundWindowsImport {
+		log.Fatal("missing expected import of internal/syscall/windows in robustio_windows.go")
+	}
+
+	for name, content := range fileContent {
+		if err := os.WriteFile(name, content, 0644); err != nil {
+			log.Fatalf("writing %q: %v", name, err)
+		}
+	}
+}
+
+// addPlusBuildConstraints splices in +build constraints for go:build
+// constraints encountered in the source.
+//
+// Gopls still builds at Go 1.16, which requires +build constraints.
+func addPlusBuildConstraints(src []byte) []byte {
+	var s scanner.Scanner
+	fset := token.NewFileSet()
+	file := fset.AddFile("", fset.Base(), len(src))
+	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
+
+	result := make([]byte, 0, len(src))
+	lastInsertion := 0
+	for {
+		pos, tok, lit := s.Scan()
+		if tok == token.EOF {
+			break
+		}
+		if tok == token.COMMENT {
+			if c, err := constraint.Parse(lit); err == nil {
+				plusBuild, err := constraint.PlusBuildLines(c)
+				if err != nil {
+					log.Fatalf("computing +build constraint for %q: %v", lit, err)
+				}
+				insertAt := file.Offset(pos) + len(lit)
+				result = append(result, src[lastInsertion:insertAt]...)
+				result = append(result, []byte("\n"+strings.Join(plusBuild, "\n"))...)
+				lastInsertion = insertAt
+			}
+		}
+	}
+	result = append(result, src[lastInsertion:]...)
+	return result
+}
diff --git a/internal/robustio/gopls_windows.go b/internal/robustio/gopls_windows.go
new file mode 100644
index 000000000..949f27816
--- /dev/null
+++ b/internal/robustio/gopls_windows.go
@@ -0,0 +1,16 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package robustio
+
+import "syscall"
+
+// The robustio package is copied from cmd/go/internal/robustio, a package used
+// by the go command to retry known flaky operations on certain operating systems.
+
+//go:generate go run copyfiles.go
+
+// Since the gopls module cannot access internal/syscall/windows, copy a
+// necessary constant.
+const ERROR_SHARING_VIOLATION syscall.Errno = 32
diff --git a/internal/robustio/robustio.go b/internal/robustio/robustio.go
new file mode 100644
index 000000000..0a559fc9b
--- /dev/null
+++ b/internal/robustio/robustio.go
@@ -0,0 +1,69 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package robustio wraps I/O functions that are prone to failure on Windows,
+// transparently retrying errors up to an arbitrary timeout.
+//
+// Errors are classified heuristically and retries are bounded, so the functions
+// in this package do not completely eliminate spurious errors. However, they do
+// significantly reduce the rate of failure in practice.
+//
+// If so, the error will likely wrap one of:
+// The functions in this package do not completely eliminate spurious errors,
+// but substantially reduce their rate of occurrence in practice.
+package robustio
+
+import "time"
+
+// Rename is like os.Rename, but on Windows retries errors that may occur if the
+// file is concurrently read or overwritten.
+//
+// (See golang.org/issue/31247 and golang.org/issue/32188.)
+func Rename(oldpath, newpath string) error {
+	return rename(oldpath, newpath)
+}
+
+// ReadFile is like os.ReadFile, but on Windows retries errors that may
+// occur if the file is concurrently replaced.
+//
+// (See golang.org/issue/31247 and golang.org/issue/32188.)
+func ReadFile(filename string) ([]byte, error) {
+	return readFile(filename)
+}
+
+// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
+// if an executable file in the directory has recently been executed.
+//
+// (See golang.org/issue/19491.)
+func RemoveAll(path string) error {
+	return removeAll(path)
+}
+
+// IsEphemeralError reports whether err is one of the errors that the functions
+// in this package attempt to mitigate.
+//
+// Errors considered ephemeral include:
+//   - syscall.ERROR_ACCESS_DENIED
+//   - syscall.ERROR_FILE_NOT_FOUND
+//   - internal/syscall/windows.ERROR_SHARING_VIOLATION
+//
+// This set may be expanded in the future; programs must not rely on the
+// non-ephemerality of any given error.
+func IsEphemeralError(err error) bool {
+	return isEphemeralError(err)
+}
+
+// A FileID uniquely identifies a file in the file system.
+//
+// If GetFileID(name1) returns the same ID as GetFileID(name2), the two file
+// names denote the same file.
+// A FileID is comparable, and thus suitable for use as a map key.
+type FileID struct {
+	device, inode uint64
+}
+
+// GetFileID returns the file system's identifier for the file, and its
+// modification time.
+// Like os.Stat, it reads through symbolic links.
+func GetFileID(filename string) (FileID, time.Time, error) { return getFileID(filename) }
diff --git a/internal/robustio/robustio_darwin.go b/internal/robustio/robustio_darwin.go
new file mode 100644
index 000000000..99fd8ebc2
--- /dev/null
+++ b/internal/robustio/robustio_darwin.go
@@ -0,0 +1,21 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package robustio
+
+import (
+	"errors"
+	"syscall"
+)
+
+const errFileNotFound = syscall.ENOENT
+
+// isEphemeralError returns true if err may be resolved by waiting.
+func isEphemeralError(err error) bool {
+	var errno syscall.Errno
+	if errors.As(err, &errno) {
+		return errno == errFileNotFound
+	}
+	return false
+}
diff --git a/internal/robustio/robustio_flaky.go b/internal/robustio/robustio_flaky.go
new file mode 100644
index 000000000..c6f997244
--- /dev/null
+++ b/internal/robustio/robustio_flaky.go
@@ -0,0 +1,93 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build windows || darwin
+// +build windows darwin
+
+package robustio
+
+import (
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"syscall"
+	"time"
+)
+
+const arbitraryTimeout = 2000 * time.Millisecond
+
+// retry retries ephemeral errors from f up to an arbitrary timeout
+// to work around filesystem flakiness on Windows and Darwin.
+func retry(f func() (err error, mayRetry bool)) error {
+	var (
+		bestErr     error
+		lowestErrno syscall.Errno
+		start       time.Time
+		nextSleep   time.Duration = 1 * time.Millisecond
+	)
+	for {
+		err, mayRetry := f()
+		if err == nil || !mayRetry {
+			return err
+		}
+
+		var errno syscall.Errno
+		if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
+			bestErr = err
+			lowestErrno = errno
+		} else if bestErr == nil {
+			bestErr = err
+		}
+
+		if start.IsZero() {
+			start = time.Now()
+		} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
+			break
+		}
+		time.Sleep(nextSleep)
+		nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
+	}
+
+	return bestErr
+}
+
+// rename is like os.Rename, but retries ephemeral errors.
+//
+// On Windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
+// MOVEFILE_REPLACE_EXISTING.
+//
+// Windows also provides a different system call, ReplaceFile,
+// that provides similar semantics, but perhaps preserves more metadata. (The
+// documentation on the differences between the two is very sparse.)
+//
+// Empirical error rates with MoveFileEx are lower under modest concurrency, so
+// for now we're sticking with what the os package already provides.
+func rename(oldpath, newpath string) (err error) {
+	return retry(func() (err error, mayRetry bool) {
+		err = os.Rename(oldpath, newpath)
+		return err, isEphemeralError(err)
+	})
+}
+
+// readFile is like os.ReadFile, but retries ephemeral errors.
+func readFile(filename string) ([]byte, error) {
+	var b []byte
+	err := retry(func() (err error, mayRetry bool) {
+		b, err = ioutil.ReadFile(filename)
+
+		// Unlike in rename, we do not retry errFileNotFound here: it can occur
+		// as a spurious error, but the file may also genuinely not exist, so the
+		// increase in robustness is probably not worth the extra latency.
+		return err, isEphemeralError(err) && !errors.Is(err, errFileNotFound)
+	})
+	return b, err
+}
+
+func removeAll(path string) error {
+	return retry(func() (err error, mayRetry bool) {
+		err = os.RemoveAll(path)
+		return err, isEphemeralError(err)
+	})
+}
diff --git a/internal/robustio/robustio_other.go b/internal/robustio/robustio_other.go
new file mode 100644
index 000000000..c11dbf9f1
--- /dev/null
+++ b/internal/robustio/robustio_other.go
@@ -0,0 +1,29 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows && !darwin
+// +build !windows,!darwin
+
+package robustio
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}
+
+func readFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}
+
+func removeAll(path string) error {
+	return os.RemoveAll(path)
+}
+
+func isEphemeralError(err error) bool {
+	return false
+}
diff --git a/internal/robustio/robustio_plan9.go b/internal/robustio/robustio_plan9.go
new file mode 100644
index 000000000..9fa4cacb5
--- /dev/null
+++ b/internal/robustio/robustio_plan9.go
@@ -0,0 +1,26 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build plan9
+// +build plan9
+
+package robustio
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getFileID(filename string) (FileID, time.Time, error) {
+	fi, err := os.Stat(filename)
+	if err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	dir := fi.Sys().(*syscall.Dir)
+	return FileID{
+		device: uint64(dir.Type)<<32 | uint64(dir.Dev),
+		inode:  dir.Qid.Path,
+	}, fi.ModTime(), nil
+}
diff --git a/internal/robustio/robustio_posix.go b/internal/robustio/robustio_posix.go
new file mode 100644
index 000000000..8aa13d027
--- /dev/null
+++ b/internal/robustio/robustio_posix.go
@@ -0,0 +1,28 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows && !plan9
+// +build !windows,!plan9
+
+// TODO(adonovan): use 'unix' tag when go1.19 can be assumed.
+
+package robustio
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getFileID(filename string) (FileID, time.Time, error) {
+	fi, err := os.Stat(filename)
+	if err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	stat := fi.Sys().(*syscall.Stat_t)
+	return FileID{
+		device: uint64(stat.Dev), // (int32 on darwin, uint64 on linux)
+		inode:  stat.Ino,
+	}, fi.ModTime(), nil
+}
diff --git a/internal/robustio/robustio_test.go b/internal/robustio/robustio_test.go
new file mode 100644
index 000000000..10244e21d
--- /dev/null
+++ b/internal/robustio/robustio_test.go
@@ -0,0 +1,88 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package robustio_test
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"golang.org/x/tools/internal/robustio"
+)
+
+func TestFileInfo(t *testing.T) {
+	// A nonexistent file has no ID.
+	nonexistent := filepath.Join(t.TempDir(), "nonexistent")
+	if _, _, err := robustio.GetFileID(nonexistent); err == nil {
+		t.Fatalf("GetFileID(nonexistent) succeeded unexpectedly")
+	}
+
+	// A regular file has an ID.
+	real := filepath.Join(t.TempDir(), "real")
+	if err := os.WriteFile(real, nil, 0644); err != nil {
+		t.Fatalf("can't create regular file: %v", err)
+	}
+	realID, realMtime, err := robustio.GetFileID(real)
+	if err != nil {
+		t.Fatalf("can't get ID of regular file: %v", err)
+	}
+
+	// Sleep so that we get a new mtime for subsequent writes.
+	time.Sleep(2 * time.Second)
+
+	// A second regular file has a different ID.
+	real2 := filepath.Join(t.TempDir(), "real2")
+	if err := os.WriteFile(real2, nil, 0644); err != nil {
+		t.Fatalf("can't create second regular file: %v", err)
+	}
+	real2ID, real2Mtime, err := robustio.GetFileID(real2)
+	if err != nil {
+		t.Fatalf("can't get ID of second regular file: %v", err)
+	}
+	if realID == real2ID {
+		t.Errorf("realID %+v == real2ID %+v", realID, real2ID)
+	}
+	if realMtime.Equal(real2Mtime) {
+		t.Errorf("realMtime %v == real2Mtime %v", realMtime, real2Mtime)
+	}
+
+	// A symbolic link has the same ID as its target.
+	if runtime.GOOS != "plan9" {
+		symlink := filepath.Join(t.TempDir(), "symlink")
+		if err := os.Symlink(real, symlink); err != nil {
+			t.Fatalf("can't create symbolic link: %v", err)
+		}
+		symlinkID, symlinkMtime, err := robustio.GetFileID(symlink)
+		if err != nil {
+			t.Fatalf("can't get ID of symbolic link: %v", err)
+		}
+		if realID != symlinkID {
+			t.Errorf("realID %+v != symlinkID %+v", realID, symlinkID)
+		}
+		if !realMtime.Equal(symlinkMtime) {
+			t.Errorf("realMtime %v != symlinkMtime %v", realMtime, symlinkMtime)
+		}
+	}
+
+	// Two hard-linked files have the same ID.
+	if runtime.GOOS != "plan9" && runtime.GOOS != "android" {
+		hardlink := filepath.Join(t.TempDir(), "hardlink")
+		if err := os.Link(real, hardlink); err != nil {
+			t.Fatal(err)
+		}
+		hardlinkID, hardlinkMtime, err := robustio.GetFileID(hardlink)
+		if err != nil {
+			t.Fatalf("can't get ID of hard link: %v", err)
+		}
+		if realID != hardlinkID {
+			t.Errorf("realID %+v != hardlinkID %+v", realID, hardlinkID)
+		}
+		if !realMtime.Equal(hardlinkMtime) {
+			t.Errorf("realMtime %v != hardlinkMtime %v", realMtime, hardlinkMtime)
+		}
+	}
+}
diff --git a/internal/robustio/robustio_windows.go b/internal/robustio/robustio_windows.go
new file mode 100644
index 000000000..616c32883
--- /dev/null
+++ b/internal/robustio/robustio_windows.go
@@ -0,0 +1,51 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package robustio
+
+import (
+	"errors"
+	"syscall"
+	"time"
+)
+
+const errFileNotFound = syscall.ERROR_FILE_NOT_FOUND
+
+// isEphemeralError returns true if err may be resolved by waiting.
+func isEphemeralError(err error) bool {
+	var errno syscall.Errno
+	if errors.As(err, &errno) {
+		switch errno {
+		case syscall.ERROR_ACCESS_DENIED,
+			syscall.ERROR_FILE_NOT_FOUND,
+			ERROR_SHARING_VIOLATION:
+			return true
+		}
+	}
+	return false
+}
+
+// Note: it may be convenient to have this helper return fs.FileInfo, but
+// implementing this is actually quite involved on Windows. Since we only
+// currently use mtime, keep it simple.
+func getFileID(filename string) (FileID, time.Time, error) {
+	filename16, err := syscall.UTF16PtrFromString(filename)
+	if err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	h, err := syscall.CreateFile(filename16, 0, 0, nil, syscall.OPEN_EXISTING, uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS), 0)
+	if err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	defer syscall.CloseHandle(h)
+	var i syscall.ByHandleFileInformation
+	if err := syscall.GetFileInformationByHandle(h, &i); err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	mtime := time.Unix(0, i.LastWriteTime.Nanoseconds())
+	return FileID{
+		device: uint64(i.VolumeSerialNumber),
+		inode:  uint64(i.FileIndexHigh)<<32 | uint64(i.FileIndexLow),
+	}, mtime, nil
+}