cmd/api: make NewWatcher populate its own package and import metadata

This partially undoes the optimizations of CL 177597, but makes up
some of the difference by caching the package list and import metadata
and making the initial calls concurrently, including in TestMain.
That reduces the critical path from two sequential 'go list'
invocations to just one (run many times concurrently), and eliminates
the need for assumptions about the consistency of the 'std' dependency
graph across platforms (and hard-coded special cases for packages that
violate those assumptions).

In the process, this simplifies and fixes TestBenchmark (which has
been silently broken since CL 164623).

This increases 'time go tool dist test api' on my workstation from
0m8.4s / 0m13.8s / 0m1.7s to 0m10.5s / 0m23.1s / 0m5.1s,
compared to 0m12.4s / 0m23.2s / 0m4.7s before CL 177597.

(That is, this change retains about half of the wall-time speedup, but
almost none of the user-time speedup.)

Tested manually using 'go test -race -bench=. cmd/api'.

Fixes #37951

Change-Id: Icd537e035e725e1ee7c41d97da5c6651233b927e
Reviewed-on: https://go-review.googlesource.com/c/go/+/224619
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alexander Rakoczy <alex@golang.org>

1 files changed, 25 insertions, 24 deletions

diff --git a/src/cmd/api/goapi_test.go b/src/cmd/api/goapi_test.go
index fc1bcc908a..282f26f708 100644
--- a/src/cmd/api/goapi_test.go
+++ b/src/cmd/api/goapi_test.go
@@ -9,16 +9,36 @@ import (
 	"flag"
 	"fmt"
 	"go/build"
-	"internal/testenv"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"testing"
 )
 
+func TestMain(m *testing.M) {
+	flag.Parse()
+	for _, c := range contexts {
+		c.Compiler = build.Default.Compiler
+	}
+
+	// Warm up the import cache in parallel.
+	var wg sync.WaitGroup
+	for _, context := range contexts {
+		context := context
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_ = NewWalker(context, filepath.Join(build.Default.GOROOT, "src"))
+		}()
+	}
+	wg.Wait()
+
+	os.Exit(m.Run())
+}
+
 var (
 	updateGolden = flag.Bool("updategolden", false, "update golden files")
 )
@@ -164,25 +184,12 @@ func TestSkipInternal(t *testing.T) {
 }
 
 func BenchmarkAll(b *testing.B) {
-	stds, err := exec.Command(testenv.GoToolPath(b), "list", "std").Output()
-	if err != nil {
-		b.Fatal(err)
-	}
-	b.ResetTimer()
-	pkgNames := strings.Fields(string(stds))
-
-	for _, c := range contexts {
-		c.Compiler = build.Default.Compiler
-	}
-
 	for i := 0; i < b.N; i++ {
 		for _, context := range contexts {
 			w := NewWalker(context, filepath.Join(build.Default.GOROOT, "src"))
-			for _, name := range pkgNames {
-				if name != "unsafe" && !strings.HasPrefix(name, "cmd/") && !internalPkg.MatchString(name) {
-					pkg, _ := w.Import(name)
-					w.export(pkg)
-				}
+			for _, name := range w.stdPackages {
+				pkg, _ := w.Import(name)
+				w.export(pkg)
 			}
 			w.Features()
 		}
@@ -190,9 +197,6 @@ func BenchmarkAll(b *testing.B) {
 }
 
 func TestIssue21181(t *testing.T) {
-	for _, c := range contexts {
-		c.Compiler = build.Default.Compiler
-	}
 	for _, context := range contexts {
 		w := NewWalker(context, "testdata/src/issue21181")
 		pkg, err := w.Import("p")
@@ -205,9 +209,6 @@ func TestIssue21181(t *testing.T) {
 }
 
 func TestIssue29837(t *testing.T) {
-	for _, c := range contexts {
-		c.Compiler = build.Default.Compiler
-	}
 	for _, context := range contexts {
 		w := NewWalker(context, "testdata/src/issue29837")
 		_, err := w.Import("p")