[dev.link] all: merge branch 'master' into dev.linkdev.link

Clean merge.

Change-Id: Ia7b2808bc649790198d34c226a61d9e569084dc5

13 files changed, 291 insertions, 206 deletions

diff --git a/src/cmd/go/internal/modfetch/cache.go b/src/cmd/go/internal/modfetch/cache.go
index e3074b775e..b7aa670250 100644
--- a/src/cmd/go/internal/modfetch/cache.go
+++ b/src/cmd/go/internal/modfetch/cache.go
@@ -10,10 +10,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/cfg"
@@ -59,7 +61,7 @@ func CachePath(m module.Version, suffix string) (string, error) {
 
 // DownloadDir returns the directory to which m should have been downloaded.
 // An error will be returned if the module path or version cannot be escaped.
-// An error satisfying errors.Is(err, os.ErrNotExist) will be returned
+// An error satisfying errors.Is(err, fs.ErrNotExist) will be returned
 // along with the directory if the directory does not exist or if the directory
 // is not completely populated.
 func DownloadDir(m module.Version) (string, error) {
@@ -106,14 +108,14 @@ func DownloadDir(m module.Version) (string, error) {
 // DownloadDirPartialError is returned by DownloadDir if a module directory
 // exists but was not completely populated.
 //
-// DownloadDirPartialError is equivalent to os.ErrNotExist.
+// DownloadDirPartialError is equivalent to fs.ErrNotExist.
 type DownloadDirPartialError struct {
 	Dir string
 	Err error
 }
 
 func (e *DownloadDirPartialError) Error() string     { return fmt.Sprintf("%s: %v", e.Dir, e.Err) }
-func (e *DownloadDirPartialError) Is(err error) bool { return err == os.ErrNotExist }
+func (e *DownloadDirPartialError) Is(err error) bool { return err == fs.ErrNotExist }
 
 // lockVersion locks a file within the module cache that guards the downloading
 // and extraction of the zipfile for the given module version.
@@ -155,16 +157,30 @@ func SideLock() (unlock func(), err error) {
 type cachingRepo struct {
 	path  string
 	cache par.Cache // cache for all operations
-	r     Repo
+
+	once     sync.Once
+	initRepo func() (Repo, error)
+	r        Repo
 }
 
-func newCachingRepo(r Repo) *cachingRepo {
+func newCachingRepo(path string, initRepo func() (Repo, error)) *cachingRepo {
 	return &cachingRepo{
-		r:    r,
-		path: r.ModulePath(),
+		path:     path,
+		initRepo: initRepo,
 	}
 }
 
+func (r *cachingRepo) repo() Repo {
+	r.once.Do(func() {
+		var err error
+		r.r, err = r.initRepo()
+		if err != nil {
+			r.r = errRepo{r.path, err}
+		}
+	})
+	return r.r
+}
+
 func (r *cachingRepo) ModulePath() string {
 	return r.path
 }
@@ -175,7 +191,7 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) {
 		err  error
 	}
 	c := r.cache.Do("versions:"+prefix, func() interface{} {
-		list, err := r.r.Versions(prefix)
+		list, err := r.repo().Versions(prefix)
 		return cached{list, err}
 	}).(cached)
 
@@ -197,7 +213,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
 			return cachedInfo{info, nil}
 		}
 
-		info, err = r.r.Stat(rev)
+		info, err = r.repo().Stat(rev)
 		if err == nil {
 			// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
 			// then save the information under the proper version, for future use.
@@ -224,7 +240,7 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
 
 func (r *cachingRepo) Latest() (*RevInfo, error) {
 	c := r.cache.Do("latest:", func() interface{} {
-		info, err := r.r.Latest()
+		info, err := r.repo().Latest()
 
 		// Save info for likely future Stat call.
 		if err == nil {
@@ -258,7 +274,7 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) {
 			return cached{text, nil}
 		}
 
-		text, err = r.r.GoMod(version)
+		text, err = r.repo().GoMod(version)
 		if err == nil {
 			if err := checkGoMod(r.path, version, text); err != nil {
 				return cached{text, err}
@@ -277,26 +293,11 @@ func (r *cachingRepo) GoMod(version string) ([]byte, error) {
 }
 
 func (r *cachingRepo) Zip(dst io.Writer, version string) error {
-	return r.r.Zip(dst, version)
-}
-
-// Stat is like Lookup(path).Stat(rev) but avoids the
-// repository path resolution in Lookup if the result is
-// already cached on local disk.
-func Stat(proxy, path, rev string) (*RevInfo, error) {
-	_, info, err := readDiskStat(path, rev)
-	if err == nil {
-		return info, nil
-	}
-	repo, err := Lookup(proxy, path)
-	if err != nil {
-		return nil, err
-	}
-	return repo.Stat(rev)
+	return r.repo().Zip(dst, version)
 }
 
-// InfoFile is like Stat but returns the name of the file containing
-// the cached information.
+// InfoFile is like Lookup(path).Stat(version) but returns the name of the file
+// containing the cached information.
 func InfoFile(path, version string) (string, error) {
 	if !semver.IsValid(version) {
 		return "", fmt.Errorf("invalid version %q", version)
@@ -307,10 +308,7 @@ func InfoFile(path, version string) (string, error) {
 	}
 
 	err := TryProxies(func(proxy string) error {
-		repo, err := Lookup(proxy, path)
-		if err == nil {
-			_, err = repo.Stat(version)
-		}
+		_, err := Lookup(proxy, path).Stat(version)
 		return err
 	})
 	if err != nil {
@@ -336,11 +334,7 @@ func GoMod(path, rev string) ([]byte, error) {
 			rev = info.Version
 		} else {
 			err := TryProxies(func(proxy string) error {
-				repo, err := Lookup(proxy, path)
-				if err != nil {
-					return err
-				}
-				info, err := repo.Stat(rev)
+				info, err := Lookup(proxy, path).Stat(rev)
 				if err == nil {
 					rev = info.Version
 				}
@@ -357,11 +351,8 @@ func GoMod(path, rev string) ([]byte, error) {
 		return data, nil
 	}
 
-	err = TryProxies(func(proxy string) error {
-		repo, err := Lookup(proxy, path)
-		if err == nil {
-			data, err = repo.GoMod(rev)
-		}
+	err = TryProxies(func(proxy string) (err error) {
+		data, err = Lookup(proxy, path).GoMod(rev)
 		return err
 	})
 	return data, err
diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go
index d85eddf767..c5fbb31b2b 100644
--- a/src/cmd/go/internal/modfetch/codehost/codehost.go
+++ b/src/cmd/go/internal/modfetch/codehost/codehost.go
@@ -11,6 +11,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -79,9 +80,8 @@ type Repo interface {
 	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
 
 	// RecentTag returns the most recent tag on rev or one of its predecessors
-	// with the given prefix and major version.
-	// An empty major string matches any major version.
-	RecentTag(rev, prefix, major string) (tag string, err error)
+	// with the given prefix. allowed may be used to filter out unwanted versions.
+	RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error)
 
 	// DescendsFrom reports whether rev or any of its ancestors has the given tag.
 	//
@@ -106,7 +106,7 @@ type FileRev struct {
 	Err  error  // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
 }
 
-// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a
+// UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a
 // revision rather than a file.
 type UnknownRevisionError struct {
 	Rev string
@@ -116,10 +116,10 @@ func (e *UnknownRevisionError) Error() string {
 	return "unknown revision " + e.Rev
 }
 func (UnknownRevisionError) Is(err error) bool {
-	return err == os.ErrNotExist
+	return err == fs.ErrNotExist
 }
 
-// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given
+// ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given
 // repository or module contains no commits.
 var ErrNoCommits error = noCommitsError{}
 
@@ -129,7 +129,7 @@ func (noCommitsError) Error() string {
 	return "no commits"
 }
 func (noCommitsError) Is(err error) bool {
-	return err == os.ErrNotExist
+	return err == fs.ErrNotExist
 }
 
 // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
diff --git a/src/cmd/go/internal/modfetch/codehost/git.go b/src/cmd/go/internal/modfetch/codehost/git.go
index 31921324a7..8abc039e7f 100644
--- a/src/cmd/go/internal/modfetch/codehost/git.go
+++ b/src/cmd/go/internal/modfetch/codehost/git.go
@@ -9,7 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"net/url"
 	"os"
 	"os/exec"
@@ -34,13 +34,13 @@ func LocalGitRepo(remote string) (Repo, error) {
 }
 
 // A notExistError wraps another error to retain its original text
-// but makes it opaquely equivalent to os.ErrNotExist.
+// but makes it opaquely equivalent to fs.ErrNotExist.
 type notExistError struct {
 	err error
 }
 
 func (e notExistError) Error() string   { return e.err.Error() }
-func (notExistError) Is(err error) bool { return err == os.ErrNotExist }
+func (notExistError) Is(err error) bool { return err == fs.ErrNotExist }
 
 const gitWorkDirType = "git3"
 
@@ -188,7 +188,7 @@ func (r *gitRepo) loadRefs() {
 		// For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL
 		// ourselves and see what code it serves.
 		if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
-			if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) {
+			if _, err := web.GetBytes(u); errors.Is(err, fs.ErrNotExist) {
 				gitErr = notExistError{gitErr}
 			}
 		}
@@ -505,7 +505,7 @@ func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
 	}
 	out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file)
 	if err != nil {
-		return nil, os.ErrNotExist
+		return nil, fs.ErrNotExist
 	}
 	return out, nil
 }
@@ -629,9 +629,9 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F
 		case "tag", "commit":
 			switch fileType {
 			default:
-				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
+				f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
 			case "missing":
-				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist}
+				f.Err = &fs.PathError{Path: tag + ":" + file, Op: "read", Err: fs.ErrNotExist}
 			case "blob":
 				f.Data = fileData
 			}
@@ -644,7 +644,7 @@ func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*F
 	return missing, nil
 }
 
-func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
+func (r *gitRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
 	info, err := r.Stat(rev)
 	if err != nil {
 		return "", err
@@ -680,7 +680,10 @@ func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
 			// NOTE: Do not replace the call to semver.Compare with semver.Max.
 			// We want to return the actual tag, not a canonicalized version of it,
 			// and semver.Max currently canonicalizes (see golang.org/issue/32700).
-			if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 {
+			if c := semver.Canonical(semtag); c == "" || !strings.HasPrefix(semtag, c) || !allowed(semtag) {
+				continue
+			}
+			if semver.Compare(semtag, highest) > 0 {
 				highest = semtag
 			}
 		}
@@ -823,12 +826,12 @@ func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser,
 	archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args)
 	if err != nil {
 		if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) {
-			return nil, os.ErrNotExist
+			return nil, fs.ErrNotExist
 		}
 		return nil, err
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(archive)), nil
+	return io.NopCloser(bytes.NewReader(archive)), nil
 }
 
 // ensureGitAttributes makes sure export-subst and export-ignore features are
@@ -859,7 +862,7 @@ func ensureGitAttributes(repoDir string) (err error) {
 		}
 	}()
 
-	b, err := ioutil.ReadAll(f)
+	b, err := io.ReadAll(f)
 	if err != nil {
 		return err
 	}
diff --git a/src/cmd/go/internal/modfetch/codehost/git_test.go b/src/cmd/go/internal/modfetch/codehost/git_test.go
index ba27c70f5a..981e3fe91f 100644
--- a/src/cmd/go/internal/modfetch/codehost/git_test.go
+++ b/src/cmd/go/internal/modfetch/codehost/git_test.go
@@ -10,6 +10,8 @@ import (
 	"flag"
 	"fmt"
 	"internal/testenv"
+	"io"
+	"io/fs"
 	"io/ioutil"
 	"log"
 	"os"
@@ -210,7 +212,7 @@ var readFileTests = []struct {
 		repo: gitrepo1,
 		rev:  "v2.3.4",
 		file: "another.txt",
-		err:  os.ErrNotExist.Error(),
+		err:  fs.ErrNotExist.Error(),
 	},
 }
 
@@ -432,7 +434,7 @@ func TestReadZip(t *testing.T) {
 			if tt.err != "" {
 				t.Fatalf("ReadZip: no error, wanted %v", tt.err)
 			}
-			zipdata, err := ioutil.ReadAll(rc)
+			zipdata, err := io.ReadAll(rc)
 			if err != nil {
 				t.Fatal(err)
 			}
diff --git a/src/cmd/go/internal/modfetch/codehost/shell.go b/src/cmd/go/internal/modfetch/codehost/shell.go
index 2762c55720..b9525adf5e 100644
--- a/src/cmd/go/internal/modfetch/codehost/shell.go
+++ b/src/cmd/go/internal/modfetch/codehost/shell.go
@@ -14,6 +14,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -115,7 +116,7 @@ func main() {
 				fmt.Fprintf(os.Stderr, "?%s\n", err)
 				continue
 			}
-			data, err := ioutil.ReadAll(rc)
+			data, err := io.ReadAll(rc)
 			rc.Close()
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "?%s\n", err)
diff --git a/src/cmd/go/internal/modfetch/codehost/vcs.go b/src/cmd/go/internal/modfetch/codehost/vcs.go
index 7284557f4b..ec97fc7e1b 100644
--- a/src/cmd/go/internal/modfetch/codehost/vcs.go
+++ b/src/cmd/go/internal/modfetch/codehost/vcs.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"internal/lazyregexp"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -377,7 +378,7 @@ func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
 
 	out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
 	if err != nil {
-		return nil, os.ErrNotExist
+		return nil, fs.ErrNotExist
 	}
 	return out, nil
 }
@@ -395,7 +396,7 @@ func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[s
 	return nil, vcsErrorf("ReadFileRevs not implemented")
 }
 
-func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
+func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
 	// We don't technically need to lock here since we're returning an error
 	// uncondititonally, but doing so anyway will help to avoid baking in
 	// lock-inversion bugs.
diff --git a/src/cmd/go/internal/modfetch/coderepo.go b/src/cmd/go/internal/modfetch/coderepo.go
index d043903336..b6bcf83f1a 100644
--- a/src/cmd/go/internal/modfetch/coderepo.go
+++ b/src/cmd/go/internal/modfetch/coderepo.go
@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path"
@@ -419,9 +420,14 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 		tagPrefix = r.codeDir + "/"
 	}
 
+	isRetracted, err := r.retractedVersions()
+	if err != nil {
+		isRetracted = func(string) bool { return false }
+	}
+
 	// tagToVersion returns the version obtained by trimming tagPrefix from tag.
-	// If the tag is invalid or a pseudo-version, tagToVersion returns an empty
-	// version.
+	// If the tag is invalid, retracted, or a pseudo-version, tagToVersion returns
+	// an empty version.
 	tagToVersion := func(tag string) (v string, tagIsCanonical bool) {
 		if !strings.HasPrefix(tag, tagPrefix) {
 			return "", false
@@ -436,6 +442,9 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 		if v == "" || !strings.HasPrefix(trimmed, v) {
 			return "", false // Invalid or incomplete version (just vX or vX.Y).
 		}
+		if isRetracted(v) {
+			return "", false
+		}
 		if v == trimmed {
 			tagIsCanonical = true
 		}
@@ -500,15 +509,24 @@ func (r *codeRepo) convert(info *codehost.RevInfo, statVers string) (*RevInfo, e
 		return checkGoMod()
 	}
 
+	// Find the highest tagged version in the revision's history, subject to
+	// major version and +incompatible constraints. Use that version as the
+	// pseudo-version base so that the pseudo-version sorts higher. Ignore
+	// retracted versions.
+	allowedMajor := func(major string) func(v string) bool {
+		return func(v string) bool {
+			return (major == "" || semver.Major(v) == major) && !isRetracted(v)
+		}
+	}
 	if pseudoBase == "" {
 		var tag string
 		if r.pseudoMajor != "" || canUseIncompatible() {
-			tag, _ = r.code.RecentTag(info.Name, tagPrefix, r.pseudoMajor)
+			tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor(r.pseudoMajor))
 		} else {
 			// Allow either v1 or v0, but not incompatible higher versions.
-			tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v1")
+			tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v1"))
 			if tag == "" {
-				tag, _ = r.code.RecentTag(info.Name, tagPrefix, "v0")
+				tag, _ = r.code.RecentTag(info.Name, tagPrefix, allowedMajor("v0"))
 			}
 		}
 		pseudoBase, _ = tagToVersion(tag) // empty if the tag is invalid
@@ -869,6 +887,57 @@ func (r *codeRepo) modPrefix(rev string) string {
 	return r.modPath + "@" + rev
 }
 
+func (r *codeRepo) retractedVersions() (func(string) bool, error) {
+	versions, err := r.Versions("")
+	if err != nil {
+		return nil, err
+	}
+
+	for i, v := range versions {
+		if strings.HasSuffix(v, "+incompatible") {
+			versions = versions[:i]
+			break
+		}
+	}
+	if len(versions) == 0 {
+		return func(string) bool { return false }, nil
+	}
+
+	var highest string
+	for i := len(versions) - 1; i >= 0; i-- {
+		v := versions[i]
+		if semver.Prerelease(v) == "" {
+			highest = v
+			break
+		}
+	}
+	if highest == "" {
+		highest = versions[len(versions)-1]
+	}
+
+	data, err := r.GoMod(highest)
+	if err != nil {
+		return nil, err
+	}
+	f, err := modfile.ParseLax("go.mod", data, nil)
+	if err != nil {
+		return nil, err
+	}
+	retractions := make([]modfile.VersionInterval, len(f.Retract))
+	for _, r := range f.Retract {
+		retractions = append(retractions, r.VersionInterval)
+	}
+
+	return func(v string) bool {
+		for _, r := range retractions {
+			if semver.Compare(r.Low, v) <= 0 && semver.Compare(v, r.High) <= 0 {
+				return true
+			}
+		}
+		return false
+	}, nil
+}
+
 func (r *codeRepo) Zip(dst io.Writer, version string) error {
 	if version != module.CanonicalVersion(version) {
 		return fmt.Errorf("version %s is not canonical", version)
@@ -972,7 +1041,7 @@ type zipFile struct {
 }
 
 func (f zipFile) Path() string                 { return f.name }
-func (f zipFile) Lstat() (os.FileInfo, error)  { return f.f.FileInfo(), nil }
+func (f zipFile) Lstat() (fs.FileInfo, error)  { return f.f.FileInfo(), nil }
 func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
 
 type dataFile struct {
@@ -981,9 +1050,9 @@ type dataFile struct {
 }
 
 func (f dataFile) Path() string                { return f.name }
-func (f dataFile) Lstat() (os.FileInfo, error) { return dataFileInfo{f}, nil }
+func (f dataFile) Lstat() (fs.FileInfo, error) { return dataFileInfo{f}, nil }
 func (f dataFile) Open() (io.ReadCloser, error) {
-	return ioutil.NopCloser(bytes.NewReader(f.data)), nil
+	return io.NopCloser(bytes.NewReader(f.data)), nil
 }
 
 type dataFileInfo struct {
@@ -992,7 +1061,7 @@ type dataFileInfo struct {
 
 func (fi dataFileInfo) Name() string       { return path.Base(fi.f.name) }
 func (fi dataFileInfo) Size() int64        { return int64(len(fi.f.data)) }
-func (fi dataFileInfo) Mode() os.FileMode  { return 0644 }
+func (fi dataFileInfo) Mode() fs.FileMode  { return 0644 }
 func (fi dataFileInfo) ModTime() time.Time { return time.Time{} }
 func (fi dataFileInfo) IsDir() bool        { return false }
 func (fi dataFileInfo) Sys() interface{}   { return nil }
diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go
index f69c193b86..4364fef6d1 100644
--- a/src/cmd/go/internal/modfetch/coderepo_test.go
+++ b/src/cmd/go/internal/modfetch/coderepo_test.go
@@ -60,7 +60,6 @@ var altVgotests = map[string]string{
 type codeRepoTest struct {
 	vcs         string
 	path        string
-	lookErr     string
 	mpath       string
 	rev         string
 	err         string
@@ -332,9 +331,9 @@ var codeRepoTests = []codeRepoTest{
 		// package in subdirectory - custom domain
 		// In general we can't reject these definitively in Lookup,
 		// but gopkg.in is special.
-		vcs:     "git",
-		path:    "gopkg.in/yaml.v2/abc",
-		lookErr: "invalid module path \"gopkg.in/yaml.v2/abc\"",
+		vcs:  "git",
+		path: "gopkg.in/yaml.v2/abc",
+		err:  "invalid module path \"gopkg.in/yaml.v2/abc\"",
 	},
 	{
 		// package in subdirectory - github
@@ -440,16 +439,7 @@ func TestCodeRepo(t *testing.T) {
 						testenv.MustHaveExecPath(t, tt.vcs)
 					}
 
-					repo, err := Lookup("direct", tt.path)
-					if tt.lookErr != "" {
-						if err != nil && err.Error() == tt.lookErr {
-							return
-						}
-						t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookErr)
-					}
-					if err != nil {
-						t.Fatalf("Lookup(%q): %v", tt.path, err)
-					}
+					repo := Lookup("direct", tt.path)
 
 					if tt.mpath == "" {
 						tt.mpath = tt.path
@@ -658,7 +648,7 @@ var codeRepoVersionsTests = []struct {
 	{
 		vcs:      "git",
 		path:     "gopkg.in/russross/blackfriday.v2",
-		versions: []string{"v2.0.0", "v2.0.1"},
+		versions: []string{"v2.0.0", "v2.0.1", "v2.1.0-pre.1"},
 	},
 	{
 		vcs:      "git",
@@ -685,10 +675,7 @@ func TestCodeRepoVersions(t *testing.T) {
 					testenv.MustHaveExecPath(t, tt.vcs)
 				}
 
-				repo, err := Lookup("direct", tt.path)
-				if err != nil {
-					t.Fatalf("Lookup(%q): %v", tt.path, err)
-				}
+				repo := Lookup("direct", tt.path)
 				list, err := repo.Versions(tt.prefix)
 				if err != nil {
 					t.Fatalf("Versions(%q): %v", tt.prefix, err)
@@ -763,10 +750,7 @@ func TestLatest(t *testing.T) {
 					testenv.MustHaveExecPath(t, tt.vcs)
 				}
 
-				repo, err := Lookup("direct", tt.path)
-				if err != nil {
-					t.Fatalf("Lookup(%q): %v", tt.path, err)
-				}
+				repo := Lookup("direct", tt.path)
 				info, err := repo.Latest()
 				if err != nil {
 					if tt.err != "" {
diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go
index 01d8f007ac..25e9fb62c1 100644
--- a/src/cmd/go/internal/modfetch/fetch.go
+++ b/src/cmd/go/internal/modfetch/fetch.go
@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -63,14 +64,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
 	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
 	defer span.Done()
 
-	// If the directory exists, and no .partial file exists, the module has
-	// already been completely extracted. .partial files may be created when a
-	// module zip directory is extracted in place instead of being extracted to a
-	// temporary directory and renamed.
 	dir, err = DownloadDir(mod)
 	if err == nil {
+		// The directory has already been completely extracted (no .partial file exists).
 		return dir, nil
-	} else if dir == "" || !errors.Is(err, os.ErrNotExist) {
+	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
 		return "", err
 	}
 
@@ -88,6 +86,9 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
 	}
 	defer unlock()
 
+	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
+	defer span.Done()
+
 	// Check whether the directory was populated while we were waiting on the lock.
 	_, dirErr := DownloadDir(mod)
 	if dirErr == nil {
@@ -95,10 +96,11 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
 	}
 	_, dirExists := dirErr.(*DownloadDirPartialError)
 
-	// Clean up any remaining temporary directories from previous runs, as well
-	// as partially extracted diectories created by future versions of cmd/go.
-	// This is only safe to do because the lock file ensures that their writers
-	// are no longer active.
+	// Clean up any remaining temporary directories created by old versions
+	// (before 1.16), as well as partially extracted directories (indicated by
+	// DownloadDirPartialError, usually because of a .partial file). This is only
+	// safe to do because the lock file ensures that their writers are no longer
+	// active.
 	parentDir := filepath.Dir(dir)
 	tmpPrefix := filepath.Base(dir) + ".tmp-"
 	if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
@@ -116,88 +118,44 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) {
 	if err != nil {
 		return "", err
 	}
-	if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) {
-		return "", err
-	}
 
-	// Extract the module zip directory.
+	// Extract the module zip directory at its final location.
 	//
-	// By default, we extract to a temporary directory, then atomically rename to
-	// its final location. We use the existence of the source directory to signal
-	// that it has been extracted successfully (see DownloadDir).  If someone
-	// deletes the entire directory (e.g., as an attempt to prune out file
-	// corruption), the module cache will still be left in a recoverable
-	// state.
+	// To prevent other processes from reading the directory if we crash,
+	// create a .partial file before extracting the directory, and delete
+	// the .partial file afterward (all while holding the lock).
 	//
-	// Unfortunately, os.Rename may fail with ERROR_ACCESS_DENIED on Windows if
-	// another process opens files in the temporary directory. This is partially
-	// mitigated by using robustio.Rename, which retries os.Rename for a short
-	// time.
+	// Before Go 1.16, we extracted to a temporary directory with a random name
+	// then renamed it into place with os.Rename. On Windows, this failed with
+	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
+	// opened files in the temporary directory.
 	//
-	// To avoid this error completely, if unzipInPlace is set, we instead create a
-	// .partial file (indicating the directory isn't fully extracted), then we
-	// extract the directory at its final location, then we delete the .partial
-	// file. This is not the default behavior because older versions of Go may
-	// simply stat the directory to check whether it exists without looking for a
-	// .partial file. If multiple versions run concurrently, the older version may
-	// assume a partially extracted directory is complete.
-	// TODO(golang.org/issue/36568): when these older versions are no longer
-	// supported, remove the old default behavior and the unzipInPlace flag.
+	// Go 1.14.2 and higher respect .partial files. Older versions may use
+	// partially extracted directories. 'go mod verify' can detect this,
+	// and 'go clean -modcache' can fix it.
 	if err := os.MkdirAll(parentDir, 0777); err != nil {
 		return "", err
 	}
-
-	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
-	if unzipInPlace {
-		if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil {
-			return "", err
-		}
-		if err := modzip.Unzip(dir, mod, zipfile); err != nil {
-			fmt.Fprintf(os.Stderr, "-> %s\n", err)
-			if rmErr := RemoveAll(dir); rmErr == nil {
-				os.Remove(partialPath)
-			}
-			return "", err
-		}
-		if err := os.Remove(partialPath); err != nil {
-			return "", err
-		}
-	} else {
-		tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
-		if err != nil {
-			return "", err
-		}
-		if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
-			fmt.Fprintf(os.Stderr, "-> %s\n", err)
-			RemoveAll(tmpDir)
-			return "", err
-		}
-		if err := robustio.Rename(tmpDir, dir); err != nil {
-			RemoveAll(tmpDir)
-			return "", err
+	if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil {
+		return "", err
+	}
+	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
+		fmt.Fprintf(os.Stderr, "-> %s\n", err)
+		if rmErr := RemoveAll(dir); rmErr == nil {
+			os.Remove(partialPath)
 		}
+		return "", err
+	}
+	if err := os.Remove(partialPath); err != nil {
+		return "", err
 	}
-	defer span.Done()
 
 	if !cfg.ModCacheRW {
-		// Make dir read-only only *after* renaming it.
-		// os.Rename was observed to fail for read-only directories on macOS.
 		makeDirsReadOnly(dir)
 	}
 	return dir, nil
 }
 
-var unzipInPlace bool
-
-func init() {
-	for _, f := range strings.Split(os.Getenv("GODEBUG"), ",") {
-		if f == "modcacheunzipinplace=1" {
-			unzipInPlace = true
-			break
-		}
-	}
-}
-
 var downloadZipCache par.Cache
 
 // DownloadZip downloads the specific module version to the
@@ -276,12 +234,28 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
 		}
 	}()
 
+	var unrecoverableErr error
 	err = TryProxies(func(proxy string) error {
-		repo, err := Lookup(proxy, mod.Path)
+		if unrecoverableErr != nil {
+			return unrecoverableErr
+		}
+		repo := Lookup(proxy, mod.Path)
+		err := repo.Zip(f, mod.Version)
 		if err != nil {
-			return err
+			// Zip may have partially written to f before failing.
+			// (Perhaps the server crashed while sending the file?)
+			// Since we allow fallback on error in some cases, we need to fix up the
+			// file to be empty again for the next attempt.
+			if _, err := f.Seek(0, io.SeekStart); err != nil {
+				unrecoverableErr = err
+				return err
+			}
+			if err := f.Truncate(0); err != nil {
+				unrecoverableErr = err
+				return err
+			}
 		}
-		return repo.Zip(f, mod.Version)
+		return err
 	})
 	if err != nil {
 		return err
@@ -341,10 +315,10 @@ func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err e
 func makeDirsReadOnly(dir string) {
 	type pathMode struct {
 		path string
-		mode os.FileMode
+		mode fs.FileMode
 	}
 	var dirs []pathMode // in lexical order
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
 		if err == nil && info.Mode()&0222 != 0 {
 			if info.IsDir() {
 				dirs = append(dirs, pathMode{path, info.Mode()})
@@ -363,7 +337,7 @@ func makeDirsReadOnly(dir string) {
 // any permission changes needed to do so.
 func RemoveAll(dir string) error {
 	// Module cache has 0555 directories; make them writable in order to remove content.
-	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return nil // ignore errors walking in file system
 		}
@@ -454,6 +428,28 @@ func readGoSum(dst map[module.Version][]string, file string, data []byte) error
 	return nil
 }
 
+// HaveSum returns true if the go.sum file contains an entry for mod.
+// The entry's hash must be generated with a known hash algorithm.
+// mod.Version may have a "/go.mod" suffix to distinguish sums for
+// .mod and .zip files.
+func HaveSum(mod module.Version) bool {
+	goSum.mu.Lock()
+	defer goSum.mu.Unlock()
+	inited, err := initGoSum()
+	if err != nil || !inited {
+		return false
+	}
+	for _, h := range goSum.m[mod] {
+		if !strings.HasPrefix(h, "h1:") {
+			continue
+		}
+		if !goSum.status[modSum{mod, h}].dirty {
+			return true
+		}
+	}
+	return false
+}
+
 // checkMod checks the given module's checksum.
 func checkMod(mod module.Version) {
 	if cfg.GOMODCACHE == "" {
@@ -468,7 +464,7 @@ func checkMod(mod module.Version) {
 	}
 	data, err := renameio.ReadFile(ziphash)
 	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
+		if errors.Is(err, fs.ErrNotExist) {
 			// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
 			return
 		}
@@ -487,7 +483,7 @@ func checkMod(mod module.Version) {
 // goModSum returns the checksum for the go.mod contents.
 func goModSum(data []byte) (string, error) {
 	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
-		return ioutil.NopCloser(bytes.NewReader(data)), nil
+		return io.NopCloser(bytes.NewReader(data)), nil
 	})
 }
 
diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go
index 4ac26650a9..d75b4da521 100644
--- a/src/cmd/go/internal/modfetch/proxy.go
+++ b/src/cmd/go/internal/modfetch/proxy.go
@@ -9,9 +9,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"io/fs"
 	"net/url"
-	"os"
 	"path"
 	pathpkg "path"
 	"path/filepath"
@@ -186,7 +185,7 @@ func proxyList() ([]proxySpec, error) {
 
 // TryProxies iterates f over each configured proxy (including "noproxy" and
 // "direct" if applicable) until f returns no error or until f returns an
-// error that is not equivalent to os.ErrNotExist on a proxy configured
+// error that is not equivalent to fs.ErrNotExist on a proxy configured
 // not to fall back on errors.
 //
 // TryProxies then returns that final error.
@@ -222,7 +221,7 @@ func TryProxies(f func(proxy string) error) error {
 		if err == nil {
 			return nil
 		}
-		isNotExistErr := errors.Is(err, os.ErrNotExist)
+		isNotExistErr := errors.Is(err, fs.ErrNotExist)
 
 		if proxy.url == "direct" || (proxy.url == "noproxy" && err != errUseProxy) {
 			bestErr = err
@@ -305,7 +304,7 @@ func (p *proxyRepo) getBytes(path string) ([]byte, error) {
 		return nil, err
 	}
 	defer body.Close()
-	return ioutil.ReadAll(body)
+	return io.ReadAll(body)
 }
 
 func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
@@ -428,7 +427,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
 func (p *proxyRepo) Latest() (*RevInfo, error) {
 	data, err := p.getBytes("@latest")
 	if err != nil {
-		if !errors.Is(err, os.ErrNotExist) {
+		if !errors.Is(err, fs.ErrNotExist) {
 			return nil, p.versionError("", err)
 		}
 		return p.latest()
diff --git a/src/cmd/go/internal/modfetch/pseudo.go b/src/cmd/go/internal/modfetch/pseudo.go
index 20c0b060ab..93eb0fad96 100644
--- a/src/cmd/go/internal/modfetch/pseudo.go
+++ b/src/cmd/go/internal/modfetch/pseudo.go
@@ -76,6 +76,12 @@ func PseudoVersion(major, older string, t time.Time, rev string) string {
 	return v + incDecimal(patch) + "-0." + segment + build
 }
 
+// ZeroPseudoVersion returns a pseudo-version with a zero timestamp and
+// revision, which may be used as a placeholder.
+func ZeroPseudoVersion(major string) string {
+	return PseudoVersion(major, "", time.Time{}, "000000000000")
+}
+
 // incDecimal returns the decimal string incremented by 1.
 func incDecimal(decimal string) string {
 	// Scan right to left turning 9s to 0s until you find a digit to increment.
@@ -120,6 +126,12 @@ func IsPseudoVersion(v string) bool {
 	return strings.Count(v, "-") >= 2 && semver.IsValid(v) && pseudoVersionRE.MatchString(v)
 }
 
+// IsZeroPseudoVersion returns whether v is a pseudo-version with a zero base,
+// timestamp, and revision, as returned by ZeroPseudoVersion.
+func IsZeroPseudoVersion(v string) bool {
+	return v == ZeroPseudoVersion(semver.Major(v))
+}
+
 // PseudoVersionTime returns the time stamp of the pseudo-version v.
 // It returns an error if v is not a pseudo-version or if the time stamp
 // embedded in the pseudo-version is not a valid time.
diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go
index eed4dd4258..af9e24cefd 100644
--- a/src/cmd/go/internal/modfetch/repo.go
+++ b/src/cmd/go/internal/modfetch/repo.go
@@ -7,6 +7,7 @@ package modfetch
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"sort"
 	"strconv"
@@ -32,8 +33,17 @@ type Repo interface {
 
 	// Versions lists all known versions with the given prefix.
 	// Pseudo-versions are not included.
+	//
 	// Versions should be returned sorted in semver order
 	// (implementations can use SortVersions).
+	//
+	// Versions returns a non-nil error only if there was a problem
+	// fetching the list of versions: it may return an empty list
+	// along with a nil error if the list of matching versions
+	// is known to be empty.
+	//
+	// If the underlying repository does not exist,
+	// Versions returns an error matching errors.Is(_, os.NotExist).
 	Versions(prefix string) ([]string, error)
 
 	// Stat returns information about the revision rev.
@@ -188,27 +198,26 @@ type lookupCacheKey struct {
 //
 // A successful return does not guarantee that the module
 // has any defined versions.
-func Lookup(proxy, path string) (Repo, error) {
+func Lookup(proxy, path string) Repo {
 	if traceRepo {
 		defer logCall("Lookup(%q, %q)", proxy, path)()
 	}
 
 	type cached struct {
-		r   Repo
-		err error
+		r Repo
 	}
 	c := lookupCache.Do(lookupCacheKey{proxy, path}, func() interface{} {
-		r, err := lookup(proxy, path)
-		if err == nil {
-			if traceRepo {
+		r := newCachingRepo(path, func() (Repo, error) {
+			r, err := lookup(proxy, path)
+			if err == nil && traceRepo {
 				r = newLoggingRepo(r)
 			}
-			r = newCachingRepo(r)
-		}
-		return cached{r, err}
+			return r, err
+		})
+		return cached{r}
 	}).(cached)
 
-	return c.r, c.err
+	return c.r
 }
 
 // lookup returns the module with the given module path.
@@ -228,7 +237,7 @@ func lookup(proxy, path string) (r Repo, err error) {
 
 	switch proxy {
 	case "off":
-		return nil, errProxyOff
+		return errRepo{path, errProxyOff}, nil
 	case "direct":
 		return lookupDirect(path)
 	case "noproxy":
@@ -407,7 +416,24 @@ func (l *loggingRepo) Zip(dst io.Writer, version string) error {
 	return l.r.Zip(dst, version)
 }
 
-// A notExistError is like os.ErrNotExist, but with a custom message
+// errRepo is a Repo that returns the same error for all operations.
+//
+// It is useful in conjunction with caching, since cache hits will not attempt
+// the prohibited operations.
+type errRepo struct {
+	modulePath string
+	err        error
+}
+
+func (r errRepo) ModulePath() string { return r.modulePath }
+
+func (r errRepo) Versions(prefix string) (tags []string, err error) { return nil, r.err }
+func (r errRepo) Stat(rev string) (*RevInfo, error)                 { return nil, r.err }
+func (r errRepo) Latest() (*RevInfo, error)                         { return nil, r.err }
+func (r errRepo) GoMod(version string) ([]byte, error)              { return nil, r.err }
+func (r errRepo) Zip(dst io.Writer, version string) error           { return r.err }
+
+// A notExistError is like fs.ErrNotExist, but with a custom message
 type notExistError struct {
 	err error
 }
@@ -421,7 +447,7 @@ func (e notExistError) Error() string {
 }
 
 func (notExistError) Is(target error) bool {
-	return target == os.ErrNotExist
+	return target == fs.ErrNotExist
 }
 
 func (e notExistError) Unwrap() error {
diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go
index 47a2571531..4fbc54d15c 100644
--- a/src/cmd/go/internal/modfetch/sumdb.go
+++ b/src/cmd/go/internal/modfetch/sumdb.go
@@ -12,7 +12,8 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"io/fs"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -182,7 +183,7 @@ func (c *dbClient) initBase() {
 			return nil
 		}
 	})
-	if errors.Is(err, os.ErrNotExist) {
+	if errors.Is(err, fs.ErrNotExist) {
 		// No proxies, or all proxies failed (with 404, 410, or were were allowed
 		// to fall back), or we reached an explicit "direct" or "off".
 		c.base = c.direct
@@ -203,7 +204,7 @@ func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
 	}
 	targ := filepath.Join(cfg.SumdbDir, file)
 	data, err = lockedfile.Read(targ)
-	if errors.Is(err, os.ErrNotExist) {
+	if errors.Is(err, fs.ErrNotExist) {
 		// Treat non-existent as empty, to bootstrap the "latest" file
 		// the first time we connect to a given database.
 		return []byte{}, nil
@@ -227,7 +228,7 @@ func (*dbClient) WriteConfig(file string, old, new []byte) error {
 		return err
 	}
 	defer f.Close()
-	data, err := ioutil.ReadAll(f)
+	data, err := io.ReadAll(f)
 	if err != nil {
 		return err
 	}
@@ -257,7 +258,7 @@ func (*dbClient) ReadCache(file string) ([]byte, error) {
 	// during which the empty file can be locked for reading.
 	// Treat observing an empty file as file not found.
 	if err == nil && len(data) == 0 {
-		err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist}
+		err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist}
 	}
 	return data, err
 }