Skip to content

cmd/dist: avoid CPU underutilization starting from GOMAXPROCS=2 runtime #65703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

cmd/dist: avoid CPU underutilization starting from GOMAXPROCS=2 runtime #65703

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
12cfb00
cmd/dist: increase test parallelism starting with GOMAXPROCS=2 runtime
qiulaidongfeng Feb 14, 2024
b2d0bb8
n
qiulaidongfeng Mar 2, 2024
d463866
n
qiulaidongfeng Mar 13, 2024
cd1e3c4
n
qiulaidongfeng Mar 14, 2024
18cffb7
n
qiulaidongfeng Mar 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 51 additions & 33 deletions src/cmd/dist/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -711,24 +711,6 @@ func (t *tester) registerTests() {
})
}

// Runtime CPU tests.
if !t.compileOnly && t.hasParallelism() {
for i := 1; i <= 4; i *= 2 {
t.registerTest(fmt.Sprintf("GOMAXPROCS=2 runtime -cpu=%d -quick", i),
&goTest{
variant: "cpu" + strconv.Itoa(i),
timeout: 300 * time.Second,
cpu: strconv.Itoa(i),
short: true,
testFlags: []string{"-quick"},
// We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
// creation of first goroutines and first garbage collections in the parallel setting.
env: []string{"GOMAXPROCS=2"},
pkg: "runtime",
})
}
}

// GOEXPERIMENT=rangefunc tests
if !t.compileOnly {
t.registerTest("GOEXPERIMENT=rangefunc go test iter",
Expand Down Expand Up @@ -864,10 +846,6 @@ func (t *tester) registerTests() {
})
}

if t.raceDetectorSupported() {
t.registerRaceTests()
}

const cgoHeading = "Testing cgo"
if t.cgoEnabled {
t.registerCgoTests(cgoHeading)
Expand All @@ -883,6 +861,40 @@ func (t *tester) registerTests() {
})
}

// Only run the API check on fast development platforms.
// Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
// so we really only need to run this check once anywhere to get adequate coverage.
// To help developers avoid trybot-only failures, we try to run on typical developer machines
// which is darwin,linux,windows/amd64 and darwin/arm64.
//
// The same logic applies to the release notes that correspond to each api/next file.
if goos == "darwin" || ((goos == "linux" || goos == "windows") && goarch == "amd64") {
t.registerTest("API release note check", &goTest{variant: "check", pkg: "cmd/relnote", testFlags: []string{"-check"}})
t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", timeout: 5 * time.Minute, testFlags: []string{"-check"}})
}

// Runtime CPU tests.
if !t.compileOnly && t.hasParallelism() {
for i := 1; i <= 4; i *= 2 {
t.registerTest(fmt.Sprintf("GOMAXPROCS=2 runtime -cpu=%d -quick", i),
&goTest{
variant: "cpu" + strconv.Itoa(i),
timeout: 300 * time.Second,
cpu: strconv.Itoa(i),
short: true,
testFlags: []string{"-quick"},
// We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
// creation of first goroutines and first garbage collections in the parallel setting.
env: []string{"GOMAXPROCS=2"},
pkg: "runtime",
})
}
}

if t.raceDetectorSupported() {
t.registerRaceTests()
}

if goos != "android" && !t.iOS() {
// Only start multiple test dir shards on builders,
// where they get distributed to multiple machines.
Expand All @@ -907,17 +919,6 @@ func (t *tester) registerTests() {
)
}
}
// Only run the API check on fast development platforms.
// Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
// so we really only need to run this check once anywhere to get adequate coverage.
// To help developers avoid trybot-only failures, we try to run on typical developer machines
// which is darwin,linux,windows/amd64 and darwin/arm64.
//
// The same logic applies to the release notes that correspond to each api/next file.
if goos == "darwin" || ((goos == "linux" || goos == "windows") && goarch == "amd64") {
t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", timeout: 5 * time.Minute, testFlags: []string{"-check"}})
t.registerTest("API release note check", &goTest{variant: "check", pkg: "cmd/relnote", testFlags: []string{"-check"}})
}
}

// addTest adds an arbitrary test callback to the test list.
Expand Down Expand Up @@ -1313,6 +1314,23 @@ func (t *tester) runPending(nextTest *distTest) {
}(w)
}

maxbg := maxbg
// for runtime.NumCPU() < 4 || runtime.GOMAXPROCS(0) == 1, do not change maxbg.
// Because there is not enough CPU to parallel the testing of multiple packages.
if runtime.NumCPU() > 4 && runtime.GOMAXPROCS(0) != 1 {
for _, w := range worklist {
// See go.dev/issue/65164
// because GOMAXPROCS=2 runtime CPU usage is low,
// so increase maxbg to avoid slowing down execution with low CPU usage.
// This makes testing a single package slower,
// but testing multiple packages together faster.
if strings.Contains(w.dt.heading, "GOMAXPROCS=2 runtime") {
maxbg = runtime.NumCPU()
break
}
}
}

started := 0
ended := 0
var last *distTest
Expand Down