dashboard: split misc-compile builders
This change splits up the misc-compile builders to only build for one
platform each. This is the first step in a sequence of CLs to simplify
misc-compile builders by eliminating the !SplitMakeRun paths, of which
misc-compile builders are the only ones.
The end result will be the support of misc-compile builders for
subrepos.
Splitting up misc-compile builders has the potential to increase VM
demand, but each build also takes less time to run. The main way is
could increase overall VM resource usage is the overhead of setting up
and tearing down a VM. The other downside is more visual noise, though
this is fairly minor.
Note that this change also passes GOOS, GOARCH, and GOARM environment
variables to buildall.sh. The goal is to eventually remove buildall.sh
by just doing make.bash. (Right now buildall.sh will essentially ignore
those variables and just set them itself.)
For golang/go#58163.
Change-Id: Ia48f6a16d080e9e078b2959dea3b68fe43def8ec
Reviewed-on: https://go-review.googlesource.com/c/build/+/463777
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
diff --git a/dashboard/builders.go b/dashboard/builders.go
index c3e8936..3166200 100644
--- a/dashboard/builders.go
+++ b/dashboard/builders.go
@@ -1519,70 +1519,89 @@
})
// addMiscCompileGo1 adds a misc-compile TryBot that
- // runs buildall.bash on the specified target(s), up to 3 max.
- // The targets are matched against the "go tool dist list" name,
- // but with hyphens instead of forward slashes ("linux-amd64", etc).
+ // runs buildall.bash on the specified target ("$goos-$goarch").
// If min is non-zero, it specifies the minimum Go 1.x version.
- addMiscCompileGo1 := func(min int, suffix string, targets ...string) {
- if len(targets) > 3 {
- // This limit will do until we have better visibility
- // into holistic TryBot completion times via metrics.
- panic("at most 3 targets may be specified to avoid making TryBots slow; see issues 32632 and 17104")
- }
+ addMiscCompileGo1 := func(min int, goos, goarch, extraSuffix string, extraEnv ...string) {
var v types.MajorMinor
var alsoNote string
if min != 0 {
v = types.MajorMinor{Major: 1, Minor: min}
alsoNote = fmt.Sprintf(" Applies to Go 1.%d and newer.", min)
}
+ // As an extra special case, add -arm5 to buildall.bash's filtering pattern for
+ // the linux-arm-arm5 case. buildall.bash treats this case specially and it explicitly
+ // unsets GOARM, so we can't just tell it what to do that way. For now, just do what
+ // it wants, but continue to pass GOARM for this case so that we can change buildall.bash
+ // safely.
+ //
+ // TODO(mknyszek): Fix buildall.bash to not explicitly unset GOARM. Consider rewriting it
+ // to better fit this new platform-per-builder model (which should simplify it a good bit).
+ filterSuffix := ""
+ if extraSuffix == "-arm5" {
+ filterSuffix = extraSuffix
+ }
+ platform := goos + "-" + goarch + extraSuffix
addBuilder(BuildConfig{
- Name: "misc-compile" + suffix,
- HostType: "host-linux-amd64-bullseye",
- tryBot: defaultTrySet(),
- env: []string{
- "GO_DISABLE_OUTBOUND_NETWORK=1",
- },
+ Name: "misc-compile-" + platform,
+ HostType: "host-linux-amd64-bullseye",
+ tryBot: defaultTrySet(),
+ env: append(extraEnv, "GO_DISABLE_OUTBOUND_NETWORK=1", "GOOS="+goos, "GOARCH="+goarch),
tryOnly: true,
MinimumGoVersion: v,
CompileOnly: true,
- Notes: "Runs buildall.bash to cross-compile & vet std+cmd packages for " + strings.Join(targets, " & ") + ", but doesn't run any tests." + alsoNote,
+ Notes: "Runs buildall.bash to cross-compile & vet std+cmd packages (or runs 'go test -c' for subrepos) for " + platform + ", but doesn't run any tests." + alsoNote,
allScriptArgs: []string{
// Filtering pattern to buildall.bash:
- "^(" + strings.Join(targets, "|") + ")$",
+ "^(" + goos + "-" + goarch + filterSuffix + ")$",
},
})
}
// addMiscCompile adds a misc-compile TryBot
// for all supported Go versions.
- addMiscCompile := func(suffix string, targets ...string) { addMiscCompileGo1(0, suffix, targets...) }
+ addMiscCompile := func(goos, goarch string) { addMiscCompileGo1(0, goos, goarch, "") }
- // Arrange so that no more than 3 ports are tested sequentially in each misc-compile
- // TryBot to avoid any individual misc-compile TryBot from becoming a bottleneck for
- // overall TryBot completion time (currently 10 minutes; see go.dev/issue/17104).
+ // To keep things simple, have each misc-compile builder handle exactly one platform.
//
- // The TestTryBotsCompileAllPorts test is used to detect any gaps in TryBot coverage
- // when new ports are added, and the misc-compile pairs below can be re-arranged.
+ // This is potentially wasteful as there could be much more VM creation overhead, but
+ // it shouldn't add any latency. It also adds some visual noise. The alternative was
+ // more complex support for subrepos; this keeps things simple by following the same
+ // general principle as all the other builders.
//
- // (In the past, we used flexible regexp patterns that matched all architectures
- // for a given GOOS value. However, over time as new architectures were added,
- // some misc-compile TryBot could become much slower than others.)
- //
- // See go.dev/issue/32632.
- addMiscCompile("-windows-arm", "windows-arm", "windows-arm64")
- addMiscCompile("-darwin", "darwin-amd64", "darwin-arm64")
- addMiscCompile("-mips", "linux-mips", "linux-mips64")
- addMiscCompile("-mipsle", "linux-mipsle", "linux-mips64le")
- addMiscCompile("-ppc", "linux-ppc64", "linux-ppc64le", "aix-ppc64")
- addMiscCompile("-freebsd", "freebsd-386", "freebsd-arm", "freebsd-arm64")
- addMiscCompile("-netbsd", "netbsd-386", "netbsd-amd64")
- addMiscCompile("-netbsd-arm", "netbsd-arm", "netbsd-arm64")
- addMiscCompile("-openbsd", "openbsd-386") // openbsd-mips64 go.dev/issue/58110
- addMiscCompile("-openbsd-arm", "openbsd-arm", "openbsd-arm64")
- addMiscCompile("-plan9", "plan9-386", "plan9-amd64", "plan9-arm")
- addMiscCompile("-solaris", "solaris-amd64", "illumos-amd64")
- addMiscCompile("-other-1", "dragonfly-amd64", "linux-loong64")
- addMiscCompile("-other-2", "linux-riscv64", "linux-s390x", "linux-arm-arm5") // 'linux-arm-arm5' is linux/arm with GOARM=5.
- addMiscCompileGo1(20, "-go1.20", "freebsd-riscv64")
+ // See https://go.dev/issue/58163 for more details.
+ addMiscCompile("windows", "arm")
+ addMiscCompile("windows", "arm64")
+ addMiscCompile("darwin", "amd64")
+ addMiscCompile("darwin", "arm64")
+ addMiscCompile("linux", "mips")
+ addMiscCompile("linux", "mips64")
+ addMiscCompile("linux", "mipsle")
+ addMiscCompile("linux", "mips64le")
+ addMiscCompile("linux", "ppc64")
+ addMiscCompile("linux", "ppc64le")
+ addMiscCompile("aix", "ppc64")
+ addMiscCompile("freebsd", "386")
+ addMiscCompile("freebsd", "arm")
+ addMiscCompile("freebsd", "arm64")
+ addMiscCompile("netbsd", "386")
+ addMiscCompile("netbsd", "amd64")
+ addMiscCompile("netbsd", "arm")
+ addMiscCompile("netbsd", "arm64")
+ addMiscCompile("openbsd", "386")
+ // openbsd-mips64 go.dev/issue/58110
+ addMiscCompile("openbsd", "arm")
+ addMiscCompile("openbsd", "arm64")
+ addMiscCompile("plan9", "386")
+ addMiscCompile("plan9", "amd64")
+ addMiscCompile("plan9", "arm")
+ addMiscCompile("solaris", "amd64")
+ addMiscCompile("illumos", "amd64")
+ addMiscCompile("dragonfly", "amd64")
+ addMiscCompile("linux", "loong64")
+ addMiscCompile("linux", "riscv64")
+ addMiscCompile("linux", "s390x")
+ addMiscCompile("linux", "arm")
+ addMiscCompileGo1(0, "linux", "arm", "-arm5", "GOARM=5")
+ addMiscCompileGo1(20, "freebsd", "riscv64", "-go1.20")
// TODO: Issue 25963, get the misc-compile trybots for Android/iOS.
// Then consider subrepos too, so "mobile" can at least be included
@@ -2996,22 +3015,35 @@
// It adds a post-submit-only builder with KnownIssue, GoDeps set to the provided values,
// and runs on a limited set of branches to get test results without potential disruption
// for contributors. It can be modified as needed when onboarding a misc-compile builder.
-func tryNewMiscCompile(suffix, rx string, knownIssue int, goDeps []string) {
+func tryNewMiscCompile(goos, goarch, extraSuffix string, knownIssue int, goDeps []string, extraEnv ...string) {
if knownIssue == 0 {
panic("tryNewMiscCompile: knownIssue parameter must be non-zero")
}
+ // As an extra special case, add -arm5 to buildall.bash's filtering pattern for
+ // the linux-arm-arm5 case. buildall.bash treats this case specially and it explicitly
+ // unsets GOARM, so we can't just tell it what to do that way. For now, just do what
+ // it wants, but continue to pass GOARM for this case so that we can change buildall.bash
+ // safely.
+ //
+ // TODO(mknyszek): Fix buildall.bash to not explicitly unset GOARM. Consider rewriting it
+ // to better fit this new platform-per-builder model (which should simplify it a good bit).
+ filterSuffix := ""
+ if extraSuffix == "-arm5" {
+ filterSuffix = extraSuffix
+ }
+ platform := goos + "-" + goarch + extraSuffix
addBuilder(BuildConfig{
- Name: "misc-compile" + suffix,
+ Name: "misc-compile" + platform,
HostType: "host-linux-amd64-bullseye",
buildsRepo: func(repo, branch, goBranch string) bool { return repo == "go" && branch == "master" },
KnownIssues: []int{knownIssue},
GoDeps: goDeps,
- env: []string{"GO_DISABLE_OUTBOUND_NETWORK=1"},
+ env: append(extraEnv, "GOOS="+goos, "GOARCH="+goarch, "GO_DISABLE_OUTBOUND_NETWORK=1"),
CompileOnly: true,
- Notes: fmt.Sprintf("Tries buildall.bash to cross-compile & vet std+cmd packages for "+rx+", but doesn't run any tests. See go.dev/issue/%d.", knownIssue),
+ Notes: fmt.Sprintf("Tries buildall.bash to cross-compile & vet std+cmd packages for "+platform+", but doesn't run any tests. See go.dev/issue/%d.", knownIssue),
allScriptArgs: []string{
// Filtering pattern to buildall.bash:
- rx,
+ "^(" + goos + "-" + goarch + filterSuffix + ")$",
},
})
}
diff --git a/dashboard/builders_test.go b/dashboard/builders_test.go
index 9d10cd5..4153532 100644
--- a/dashboard/builders_test.go
+++ b/dashboard/builders_test.go
@@ -105,21 +105,39 @@
"windows-386-2012",
"windows-amd64-2016",
- "misc-compile-darwin",
- "misc-compile-freebsd",
"misc-compile-windows-arm",
- "misc-compile-mips",
- "misc-compile-mipsle",
- "misc-compile-netbsd",
+ "misc-compile-windows-arm64",
+ "misc-compile-darwin-amd64",
+ "misc-compile-darwin-arm64",
+ "misc-compile-linux-mips",
+ "misc-compile-linux-mips64",
+ "misc-compile-linux-mipsle",
+ "misc-compile-linux-mips64le",
+ "misc-compile-linux-ppc64",
+ "misc-compile-linux-ppc64le",
+ "misc-compile-aix-ppc64",
+ "misc-compile-freebsd-386",
+ "misc-compile-freebsd-arm",
+ "misc-compile-freebsd-arm64",
+ "misc-compile-netbsd-386",
+ "misc-compile-netbsd-amd64",
"misc-compile-netbsd-arm",
- "misc-compile-openbsd",
+ "misc-compile-netbsd-arm64",
+ "misc-compile-openbsd-386",
"misc-compile-openbsd-arm",
- "misc-compile-plan9",
- "misc-compile-ppc",
- "misc-compile-solaris",
- "misc-compile-other-1",
- "misc-compile-other-2",
- "misc-compile-go1.20",
+ "misc-compile-openbsd-arm64",
+ "misc-compile-plan9-386",
+ "misc-compile-plan9-amd64",
+ "misc-compile-plan9-arm",
+ "misc-compile-solaris-amd64",
+ "misc-compile-illumos-amd64",
+ "misc-compile-dragonfly-amd64",
+ "misc-compile-linux-loong64",
+ "misc-compile-linux-riscv64",
+ "misc-compile-linux-s390x",
+ "misc-compile-linux-arm",
+ "misc-compile-linux-arm-arm5",
+ "misc-compile-freebsd-riscv64-go1.20",
},
},
{
@@ -138,21 +156,39 @@
"windows-386-2012",
"windows-amd64-2016",
- "misc-compile-darwin",
- "misc-compile-freebsd",
"misc-compile-windows-arm",
- "misc-compile-mips",
- "misc-compile-mipsle",
- "misc-compile-netbsd",
+ "misc-compile-windows-arm64",
+ "misc-compile-darwin-amd64",
+ "misc-compile-darwin-arm64",
+ "misc-compile-linux-mips",
+ "misc-compile-linux-mips64",
+ "misc-compile-linux-mipsle",
+ "misc-compile-linux-mips64le",
+ "misc-compile-linux-ppc64",
+ "misc-compile-linux-ppc64le",
+ "misc-compile-aix-ppc64",
+ "misc-compile-freebsd-386",
+ "misc-compile-freebsd-arm",
+ "misc-compile-freebsd-arm64",
+ "misc-compile-netbsd-386",
+ "misc-compile-netbsd-amd64",
"misc-compile-netbsd-arm",
- "misc-compile-openbsd",
+ "misc-compile-netbsd-arm64",
+ "misc-compile-openbsd-386",
"misc-compile-openbsd-arm",
- "misc-compile-plan9",
- "misc-compile-ppc",
- "misc-compile-solaris",
- "misc-compile-other-1",
- "misc-compile-other-2",
- "misc-compile-go1.20",
+ "misc-compile-openbsd-arm64",
+ "misc-compile-plan9-386",
+ "misc-compile-plan9-amd64",
+ "misc-compile-plan9-arm",
+ "misc-compile-solaris-amd64",
+ "misc-compile-illumos-amd64",
+ "misc-compile-dragonfly-amd64",
+ "misc-compile-linux-loong64",
+ "misc-compile-linux-riscv64",
+ "misc-compile-linux-s390x",
+ "misc-compile-linux-arm",
+ "misc-compile-linux-arm-arm5",
+ "misc-compile-freebsd-riscv64-go1.20",
// Include longtest builders on Go repo release branches. See issue 37827.
"linux-386-longtest",
@@ -179,21 +215,39 @@
"windows-386-2012",
"windows-amd64-2016",
- "misc-compile-darwin",
- "misc-compile-freebsd",
"misc-compile-windows-arm",
- "misc-compile-mips",
- "misc-compile-mipsle",
- "misc-compile-netbsd",
+ "misc-compile-windows-arm64",
+ "misc-compile-darwin-amd64",
+ "misc-compile-darwin-arm64",
+ "misc-compile-linux-mips",
+ "misc-compile-linux-mips64",
+ "misc-compile-linux-mipsle",
+ "misc-compile-linux-mips64le",
+ "misc-compile-linux-ppc64",
+ "misc-compile-linux-ppc64le",
+ "misc-compile-aix-ppc64",
+ "misc-compile-freebsd-386",
+ "misc-compile-freebsd-arm",
+ "misc-compile-freebsd-arm64",
+ "misc-compile-netbsd-386",
+ "misc-compile-netbsd-amd64",
"misc-compile-netbsd-arm",
- "misc-compile-openbsd",
+ "misc-compile-netbsd-arm64",
+ "misc-compile-openbsd-386",
"misc-compile-openbsd-arm",
- "misc-compile-plan9",
- "misc-compile-ppc",
- "misc-compile-solaris",
- "misc-compile-other-1",
- "misc-compile-other-2",
- "misc-compile-go1.20",
+ "misc-compile-openbsd-arm64",
+ "misc-compile-plan9-386",
+ "misc-compile-plan9-amd64",
+ "misc-compile-plan9-arm",
+ "misc-compile-solaris-amd64",
+ "misc-compile-illumos-amd64",
+ "misc-compile-dragonfly-amd64",
+ "misc-compile-linux-loong64",
+ "misc-compile-linux-riscv64",
+ "misc-compile-linux-s390x",
+ "misc-compile-linux-arm",
+ "misc-compile-linux-arm-arm5",
+ "misc-compile-freebsd-riscv64-go1.20",
// Include longtest builders on Go repo release branches. See issue 37827.
"linux-386-longtest",
@@ -218,20 +272,38 @@
"windows-386-2012-oldcc",
"windows-amd64-2016-oldcc",
- "misc-compile-darwin",
- "misc-compile-freebsd",
"misc-compile-windows-arm",
- "misc-compile-mips",
- "misc-compile-mipsle",
- "misc-compile-netbsd",
+ "misc-compile-windows-arm64",
+ "misc-compile-darwin-amd64",
+ "misc-compile-darwin-arm64",
+ "misc-compile-linux-mips",
+ "misc-compile-linux-mips64",
+ "misc-compile-linux-mipsle",
+ "misc-compile-linux-mips64le",
+ "misc-compile-linux-ppc64",
+ "misc-compile-linux-ppc64le",
+ "misc-compile-aix-ppc64",
+ "misc-compile-freebsd-386",
+ "misc-compile-freebsd-arm",
+ "misc-compile-freebsd-arm64",
+ "misc-compile-netbsd-386",
+ "misc-compile-netbsd-amd64",
"misc-compile-netbsd-arm",
- "misc-compile-openbsd",
+ "misc-compile-netbsd-arm64",
+ "misc-compile-openbsd-386",
"misc-compile-openbsd-arm",
- "misc-compile-plan9",
- "misc-compile-ppc",
- "misc-compile-solaris",
- "misc-compile-other-1",
- "misc-compile-other-2",
+ "misc-compile-openbsd-arm64",
+ "misc-compile-plan9-386",
+ "misc-compile-plan9-amd64",
+ "misc-compile-plan9-arm",
+ "misc-compile-solaris-amd64",
+ "misc-compile-illumos-amd64",
+ "misc-compile-dragonfly-amd64",
+ "misc-compile-linux-loong64",
+ "misc-compile-linux-riscv64",
+ "misc-compile-linux-s390x",
+ "misc-compile-linux-arm",
+ "misc-compile-linux-arm-arm5",
// Include longtest builders on Go repo release branches. See issue 37827.
"linux-386-longtest",
@@ -906,8 +978,16 @@
t.Fatalf("invalid misc-compile filtering pattern for builder %q: %q",
b.Name, b.allScriptArgs[0])
}
- if re.MatchString("linux-arm-arm5") {
- ok = true
+ if !re.MatchString("linux-arm-arm5") {
+ continue
+ }
+ for _, v := range b.env {
+ if v == "GOARM=5" {
+ ok = true
+ break
+ }
+ }
+ if ok {
break
}
}
